From 1d28d613b1d0447b60de2d2044aeac3e1d543aa6 Mon Sep 17 00:00:00 2001 From: Bouke van der Bijl Date: Wed, 25 Oct 2023 11:39:18 +0200 Subject: [PATCH 001/141] config: add included files into parsedContents before applying Fixes #8719 --- src/libutil/config.cc | 16 ++++++++++++---- tests/functional/init.sh | 3 ++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 8e06273ee..17380b6d8 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -88,10 +88,9 @@ void Config::getSettings(std::map & res, bool overridd res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description}); } -void AbstractConfig::applyConfig(const std::string & contents, const std::string & path) { - unsigned int pos = 0; - std::vector> parsedContents; +static void applyConfigInner(const std::string & contents, const std::string & path, std::vector> & parsedContents) { + unsigned int pos = 0; while (pos < contents.size()) { std::string line; @@ -123,7 +122,10 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); auto p = absPath(tokens[1], dirOf(path)); if (pathExists(p)) { - applyConfigFile(p); + try { + std::string includedContents = readFile(path); + applyConfigInner(includedContents, p, parsedContents); + } catch (SysError &) { } } else if (!ignoreMissing) { throw Error("file '%1%' included from '%2%' not found", p, path); } @@ -143,6 +145,12 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string concatStringsSep(" ", Strings(i, tokens.end())), }); }; +} + +void AbstractConfig::applyConfig(const std::string & contents, const std::string & path) { + std::vector> parsedContents; + + applyConfigInner(contents, path, parsedContents); // First apply experimental-feature related settings for (auto & [name, value] : parsedContents) diff --git a/tests/functional/init.sh b/tests/functional/init.sh index c420e8c9f..d697b1a30 100755 --- a/tests/functional/init.sh +++ b/tests/functional/init.sh @@ -20,7 +20,7 @@ cat > "$NIX_CONF_DIR"/nix.conf < "$NIX_CONF_DIR"/nix.conf.extra < Date: Fri, 6 Jan 2023 18:06:03 -0500 Subject: [PATCH 002/141] Factor out `StoreDirConfig` More progress on #5729. --- src/libstore/path.cc | 14 ++-- src/libstore/store-api.cc | 24 +++--- src/libstore/store-api.hh | 103 +------------------------ src/libstore/store-dir-config.hh | 126 +++++++++++++++++++++++++++++++ 4 files changed, 148 insertions(+), 119 deletions(-) create mode 100644 src/libstore/store-dir-config.hh diff --git a/src/libstore/path.cc b/src/libstore/path.cc index ec3e53232..69f6d7356 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -1,4 +1,4 @@ -#include "store-api.hh" +#include "store-dir-config.hh" #include @@ -54,7 +54,7 @@ StorePath StorePath::random(std::string_view name) return StorePath(hash, name); } -StorePath Store::parseStorePath(std::string_view path) const +StorePath StoreDirConfig::parseStorePath(std::string_view path) const { auto p = canonPath(std::string(path)); if (dirOf(p) != storeDir) @@ -62,7 +62,7 @@ StorePath Store::parseStorePath(std::string_view path) const return StorePath(baseNameOf(p)); } -std::optional Store::maybeParseStorePath(std::string_view path) const +std::optional StoreDirConfig::maybeParseStorePath(std::string_view path) const { try { return parseStorePath(path); @@ -71,24 +71,24 @@ std::optional Store::maybeParseStorePath(std::string_view path) const } } -bool Store::isStorePath(std::string_view path) const +bool StoreDirConfig::isStorePath(std::string_view path) const { return (bool) maybeParseStorePath(path); } -StorePathSet Store::parseStorePathSet(const PathSet & paths) const +StorePathSet StoreDirConfig::parseStorePathSet(const PathSet & paths) const { StorePathSet res; for (auto & i : paths) res.insert(parseStorePath(i)); return res; } -std::string Store::printStorePath(const StorePath & path) const +std::string StoreDirConfig::printStorePath(const StorePath & path) const { return (storeDir + "/").append(path.to_string()); } -PathSet Store::printStorePathSet(const StorePathSet & paths) const +PathSet StoreDirConfig::printStorePathSet(const StorePathSet & paths) const { PathSet res; for (auto & i : paths) res.insert(printStorePath(i)); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 646b0ec7d..a681bb6cf 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -23,13 +23,13 @@ using json = nlohmann::json; namespace nix { -bool Store::isInStore(PathView path) const +bool StoreDirConfig::isInStore(PathView path) const { return isInDir(path, storeDir); } -std::pair Store::toStorePath(PathView path) const +std::pair StoreDirConfig::toStorePath(PathView path) const { if (!isInStore(path)) throw Error("path '%1%' is not in the Nix store", path); @@ -143,7 +143,7 @@ StorePath Store::followLinksToStorePath(std::string_view path) const */ -StorePath Store::makeStorePath(std::string_view type, +StorePath StoreDirConfig::makeStorePath(std::string_view type, std::string_view hash, std::string_view name) const { /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ @@ -154,14 +154,14 @@ StorePath Store::makeStorePath(std::string_view type, } -StorePath Store::makeStorePath(std::string_view type, +StorePath StoreDirConfig::makeStorePath(std::string_view type, const Hash & hash, std::string_view name) const { return makeStorePath(type, hash.to_string(HashFormat::Base16, true), name); } -StorePath Store::makeOutputPath(std::string_view id, +StorePath StoreDirConfig::makeOutputPath(std::string_view id, const Hash & hash, std::string_view name) const { return makeStorePath("output:" + std::string { id }, hash, outputPathName(name, id)); @@ -172,7 +172,7 @@ StorePath Store::makeOutputPath(std::string_view id, hacky, but we can't put them in, say, (per the grammar above) since that would be ambiguous. */ static std::string makeType( - const Store & store, + const StoreDirConfig & store, std::string && type, const StoreReferences & references) { @@ -185,7 +185,7 @@ static std::string makeType( } -StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const +StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) { return makeStorePath(makeType(*this, "source", info.references), info.hash, name); @@ -201,7 +201,7 @@ StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInf } -StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const +StorePath StoreDirConfig::makeTextPath(std::string_view name, const TextInfo & info) const { assert(info.hash.type == htSHA256); return makeStorePath( @@ -214,7 +214,7 @@ StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) cons } -StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const +StorePath StoreDirConfig::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const { // New template return std::visit(overloaded { @@ -228,7 +228,7 @@ StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentA } -std::pair Store::computeStorePathFromDump( +std::pair StoreDirConfig::computeStorePathFromDump( Source & dump, std::string_view name, FileIngestionMethod method, @@ -247,7 +247,7 @@ std::pair Store::computeStorePathFromDump( } -StorePath Store::computeStorePathForText( +StorePath StoreDirConfig::computeStorePathForText( std::string_view name, std::string_view s, const StorePathSet & references) const @@ -1315,7 +1315,7 @@ std::optional decodeValidPathInfo(const Store & store, std::istre } -std::string Store::showPaths(const StorePathSet & paths) +std::string StoreDirConfig::showPaths(const StorePathSet & paths) { std::string s; for (auto & i : paths) { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 6aa317e3d..bee5ec16c 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -14,6 +14,7 @@ #include "config.hh" #include "path-info.hh" #include "repair-flag.hh" +#include "store-dir-config.hh" #include #include @@ -64,7 +65,6 @@ MakeError(InvalidPath, Error); MakeError(Unsupported, Error); MakeError(SubstituteGone, Error); MakeError(SubstituterDisabled, Error); -MakeError(BadStorePath, Error); MakeError(InvalidStoreURI, Error); @@ -97,11 +97,11 @@ struct KeyedBuildResult; typedef std::map> StorePathCAMap; -struct StoreConfig : public Config +struct StoreConfig : public StoreDirConfig { typedef std::map Params; - using Config::Config; + using StoreDirConfig::StoreDirConfig; StoreConfig() = delete; @@ -131,15 +131,6 @@ struct StoreConfig : public Config return std::nullopt; } - const PathSetting storeDir_{this, settings.nixStore, - "store", - R"( - Logical location of the Nix store, usually - `/nix/store`. Note that you can only copy store paths - between stores if they have the same `store` setting. - )"}; - const Path storeDir = storeDir_; - const Setting pathInfoCacheSize{this, 65536, "path-info-cache-size", "Size of the in-memory store path metadata cache."}; @@ -224,45 +215,6 @@ public: virtual std::string getUri() = 0; - StorePath parseStorePath(std::string_view path) const; - - std::optional maybeParseStorePath(std::string_view path) const; - - std::string printStorePath(const StorePath & path) const; - - /** - * Deprecated - * - * \todo remove - */ - StorePathSet parseStorePathSet(const PathSet & paths) const; - - PathSet printStorePathSet(const StorePathSet & path) const; - - /** - * Display a set of paths in human-readable form (i.e., between quotes - * and separated by commas). - */ - std::string showPaths(const StorePathSet & paths); - - /** - * @return true if ‘path’ is in the Nix store (but not the Nix - * store itself). - */ - bool isInStore(PathView path) const; - - /** - * @return true if ‘path’ is a store path, i.e. a direct child of the - * Nix store. - */ - bool isStorePath(std::string_view path) const; - - /** - * Split a path like /nix/store/-/ into - * /nix/store/- and /. - */ - std::pair toStorePath(PathView path) const; - /** * Follow symlinks until we end up with a path in the Nix store. */ @@ -274,55 +226,6 @@ public: */ StorePath followLinksToStorePath(std::string_view path) const; - /** - * Constructs a unique store path name. - */ - StorePath makeStorePath(std::string_view type, - std::string_view hash, std::string_view name) const; - StorePath makeStorePath(std::string_view type, - const Hash & hash, std::string_view name) const; - - StorePath makeOutputPath(std::string_view id, - const Hash & hash, std::string_view name) const; - - StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const; - - StorePath makeTextPath(std::string_view name, const TextInfo & info) const; - - StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const; - - /** - * Read-only variant of addToStoreFromDump(). It returns the store - * path to which a NAR or flat file would be written. - */ - std::pair computeStorePathFromDump( - Source & dump, - std::string_view name, - FileIngestionMethod method = FileIngestionMethod::Recursive, - HashType hashAlgo = htSHA256, - const StorePathSet & references = {}) const; - - /** - * Preparatory part of addTextToStore(). - * - * !!! Computation of the path should take the references given to - * addTextToStore() into account, otherwise we have a (relatively - * minor) security hole: a caller can register a source file with - * bogus references. If there are too many references, the path may - * not be garbage collected when it has to be (not really a problem, - * the caller could create a root anyway), or it may be garbage - * collected when it shouldn't be (more serious). - * - * Hashing the references would solve this (bogus references would - * simply yield a different store path, so other users wouldn't be - * affected), but it has some backwards compatibility issues (the - * hashing scheme changes), so I'm not doing that for now. - */ - StorePath computeStorePathForText( - std::string_view name, - std::string_view s, - const StorePathSet & references) const; - /** * Check whether a path is valid. */ diff --git a/src/libstore/store-dir-config.hh b/src/libstore/store-dir-config.hh new file mode 100644 index 000000000..53843d663 --- /dev/null +++ b/src/libstore/store-dir-config.hh @@ -0,0 +1,126 @@ +#pragma once + +#include "path.hh" +#include "hash.hh" +#include "content-address.hh" +#include "globals.hh" +#include "config.hh" + +#include +#include +#include + + +namespace nix { + +MakeError(BadStorePath, Error); + +struct StoreDirConfig : public Config +{ + using Config::Config; + + StoreDirConfig() = delete; + + virtual ~StoreDirConfig() = default; + + const PathSetting storeDir_{this, settings.nixStore, + "store", + R"( + Logical location of the Nix store, usually + `/nix/store`. Note that you can only copy store paths + between stores if they have the same `store` setting. + )"}; + const Path storeDir = storeDir_; + + // pure methods + + StorePath parseStorePath(std::string_view path) const; + + std::optional maybeParseStorePath(std::string_view path) const; + + std::string printStorePath(const StorePath & path) const; + + /** + * Deprecated + * + * \todo remove + */ + StorePathSet parseStorePathSet(const PathSet & paths) const; + + PathSet printStorePathSet(const StorePathSet & path) const; + + /** + * Display a set of paths in human-readable form (i.e., between quotes + * and separated by commas). + */ + std::string showPaths(const StorePathSet & paths); + + /** + * @return true if ‘path’ is in the Nix store (but not the Nix + * store itself). + */ + bool isInStore(PathView path) const; + + /** + * @return true if ‘path’ is a store path, i.e. a direct child of the + * Nix store. + */ + bool isStorePath(std::string_view path) const; + + /** + * Split a path like /nix/store/-/ into + * /nix/store/- and /. + */ + std::pair toStorePath(PathView path) const; + + /** + * Constructs a unique store path name. + */ + StorePath makeStorePath(std::string_view type, + std::string_view hash, std::string_view name) const; + StorePath makeStorePath(std::string_view type, + const Hash & hash, std::string_view name) const; + + StorePath makeOutputPath(std::string_view id, + const Hash & hash, std::string_view name) const; + + StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const; + + StorePath makeTextPath(std::string_view name, const TextInfo & info) const; + + StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const; + + /** + * Read-only variant of addToStoreFromDump(). It returns the store + * path to which a NAR or flat file would be written. + */ + std::pair computeStorePathFromDump( + Source & dump, + std::string_view name, + FileIngestionMethod method = FileIngestionMethod::Recursive, + HashType hashAlgo = htSHA256, + const StorePathSet & references = {}) const; + + /** + * Preparatory part of addTextToStore(). + * + * !!! Computation of the path should take the references given to + * addTextToStore() into account, otherwise we have a (relatively + * minor) security hole: a caller can register a source file with + * bogus references. If there are too many references, the path may + * not be garbage collected when it has to be (not really a problem, + * the caller could create a root anyway), or it may be garbage + * collected when it shouldn't be (more serious). + * + * Hashing the references would solve this (bogus references would + * simply yield a different store path, so other users wouldn't be + * affected), but it has some backwards compatibility issues (the + * hashing scheme changes), so I'm not doing that for now. + */ + StorePath computeStorePathForText( + std::string_view name, + std::string_view s, + const StorePathSet & references) const; +}; + +} From dde1d863388617b3a63db808c125f274c86a3222 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 18 Mar 2022 15:35:45 +0000 Subject: [PATCH 003/141] Restrict some code to `StoreDirConfig` - part of eval cache - part of derivations - derived path - store path with outputs - serializers --- perl/lib/Nix/Store.xs | 1 + src/libcmd/built-path.cc | 8 ++-- src/libcmd/built-path.hh | 18 ++++----- src/libexpr/eval-cache.cc | 6 +-- src/libexpr/primops/fetchClosure.cc | 1 + src/libexpr/value.hh | 1 - src/libstore/builtins/buildenv.cc | 1 + src/libstore/builtins/buildenv.hh | 1 - src/libstore/common-protocol-impl.hh | 4 +- src/libstore/common-protocol.cc | 28 ++++++------- src/libstore/common-protocol.hh | 8 ++-- src/libstore/derivations.cc | 32 +++++++-------- src/libstore/derivations.hh | 26 +++++++------ src/libstore/derived-path.cc | 39 ++++++++++--------- src/libstore/derived-path.hh | 37 ++++++++++-------- .../length-prefixed-protocol-helper.hh | 22 +++++------ src/libstore/local-store.cc | 1 + src/libstore/misc.cc | 1 + src/libstore/path-with-outputs.cc | 4 +- src/libstore/path-with-outputs.hh | 10 +++-- src/libstore/serve-protocol-impl.hh | 8 ++-- src/libstore/serve-protocol.cc | 4 +- src/libstore/serve-protocol.hh | 12 +++--- src/libstore/store-api.cc | 2 + src/libstore/store-api.hh | 11 ++++-- src/libstore/worker-protocol-impl.hh | 8 ++-- src/libstore/worker-protocol.cc | 24 ++++++------ src/libstore/worker-protocol.hh | 12 +++--- src/nix-build/nix-build.cc | 1 + src/nix-copy-closure/nix-copy-closure.cc | 1 + 30 files changed, 175 insertions(+), 157 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 08f812b31..210d50b6e 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -9,6 +9,7 @@ #undef do_close #include "derivations.hh" +#include "realisation.hh" #include "globals.hh" #include "store-api.hh" #include "util.hh" diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index 9a2dce806..8e2efc7c3 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -80,7 +80,7 @@ SingleDerivedPath SingleBuiltPath::discardOutputPath() const ); } -nlohmann::json BuiltPath::Built::toJSON(const Store & store) const +nlohmann::json BuiltPath::Built::toJSON(const StoreDirConfig & store) const { nlohmann::json res; res["drvPath"] = drvPath->toJSON(store); @@ -90,7 +90,7 @@ nlohmann::json BuiltPath::Built::toJSON(const Store & store) const return res; } -nlohmann::json SingleBuiltPath::Built::toJSON(const Store & store) const +nlohmann::json SingleBuiltPath::Built::toJSON(const StoreDirConfig & store) const { nlohmann::json res; res["drvPath"] = drvPath->toJSON(store); @@ -100,14 +100,14 @@ nlohmann::json SingleBuiltPath::Built::toJSON(const Store & store) const return res; } -nlohmann::json SingleBuiltPath::toJSON(const Store & store) const +nlohmann::json SingleBuiltPath::toJSON(const StoreDirConfig & store) const { return std::visit([&](const auto & buildable) { return buildable.toJSON(store); }, raw()); } -nlohmann::json BuiltPath::toJSON(const Store & store) const +nlohmann::json BuiltPath::toJSON(const StoreDirConfig & store) const { return std::visit([&](const auto & buildable) { return buildable.toJSON(store); diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh index e677bc810..51918f96c 100644 --- a/src/libcmd/built-path.hh +++ b/src/libcmd/built-path.hh @@ -11,9 +11,9 @@ struct SingleBuiltPathBuilt { SingleDerivedPathBuilt discardOutputPath() const; - std::string to_string(const Store & store) const; - static SingleBuiltPathBuilt parse(const Store & store, std::string_view, std::string_view); - nlohmann::json toJSON(const Store & store) const; + std::string to_string(const StoreDirConfig & store) const; + static SingleBuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view); + nlohmann::json toJSON(const StoreDirConfig & store) const; DECLARE_CMP(SingleBuiltPathBuilt); }; @@ -38,8 +38,8 @@ struct SingleBuiltPath : _SingleBuiltPathRaw { SingleDerivedPath discardOutputPath() const; - static SingleBuiltPath parse(const Store & store, std::string_view); - nlohmann::json toJSON(const Store & store) const; + static SingleBuiltPath parse(const StoreDirConfig & store, std::string_view); + nlohmann::json toJSON(const StoreDirConfig & store) const; }; static inline ref staticDrv(StorePath drvPath) @@ -56,9 +56,9 @@ struct BuiltPathBuilt { ref drvPath; std::map outputs; - std::string to_string(const Store & store) const; - static BuiltPathBuilt parse(const Store & store, std::string_view, std::string_view); - nlohmann::json toJSON(const Store & store) const; + std::string to_string(const StoreDirConfig & store) const; + static BuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view); + nlohmann::json toJSON(const StoreDirConfig & store) const; DECLARE_CMP(BuiltPathBuilt); }; @@ -86,7 +86,7 @@ struct BuiltPath : _BuiltPathRaw { StorePathSet outPaths() const; RealisedPath::Set toRealisedPaths(Store & store) const; - nlohmann::json toJSON(const Store & store) const; + nlohmann::json toJSON(const StoreDirConfig & store) const; }; typedef std::vector BuiltPaths; diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 10fc799a9..824f94ba1 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -21,7 +21,7 @@ struct AttrDb { std::atomic_bool failed{false}; - const Store & cfg; + const StoreDirConfig & cfg; struct State { @@ -38,7 +38,7 @@ struct AttrDb SymbolTable & symbols; AttrDb( - const Store & cfg, + const StoreDirConfig & cfg, const Hash & fingerprint, SymbolTable & symbols) : cfg(cfg) @@ -322,7 +322,7 @@ struct AttrDb }; static std::shared_ptr makeAttrDb( - const Store & cfg, + const StoreDirConfig & cfg, const Hash & fingerprint, SymbolTable & symbols) { diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index b86ef6b93..27147a5d1 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -1,5 +1,6 @@ #include "primops.hh" #include "store-api.hh" +#include "realisation.hh" #include "make-content-addressed.hh" #include "url.hh" diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 622e613ea..20f268a3e 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -66,7 +66,6 @@ class Symbol; class PosIdx; struct Pos; class StorePath; -class Store; class EvalState; class XMLWriter; diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index c8911d153..9283251ac 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -1,4 +1,5 @@ #include "buildenv.hh" +#include "derivations.hh" #include #include diff --git a/src/libstore/builtins/buildenv.hh b/src/libstore/builtins/buildenv.hh index 0923c2adb..8bebd390d 100644 --- a/src/libstore/builtins/buildenv.hh +++ b/src/libstore/builtins/buildenv.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include "derivations.hh" #include "store-api.hh" namespace nix { diff --git a/src/libstore/common-protocol-impl.hh b/src/libstore/common-protocol-impl.hh index 079c182b8..360882c02 100644 --- a/src/libstore/common-protocol-impl.hh +++ b/src/libstore/common-protocol-impl.hh @@ -16,11 +16,11 @@ namespace nix { /* protocol-agnostic templates */ #define COMMON_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \ - TEMPLATE T CommonProto::Serialise< T >::read(const Store & store, CommonProto::ReadConn conn) \ + TEMPLATE T CommonProto::Serialise< T >::read(const StoreDirConfig & store, CommonProto::ReadConn conn) \ { \ return LengthPrefixedProtoHelper::read(store, conn); \ } \ - TEMPLATE void CommonProto::Serialise< T >::write(const Store & store, CommonProto::WriteConn conn, const T & t) \ + TEMPLATE void CommonProto::Serialise< T >::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const T & t) \ { \ LengthPrefixedProtoHelper::write(store, conn, t); \ } diff --git a/src/libstore/common-protocol.cc b/src/libstore/common-protocol.cc index f906814bc..c14a6cfcd 100644 --- a/src/libstore/common-protocol.cc +++ b/src/libstore/common-protocol.cc @@ -14,40 +14,40 @@ namespace nix { /* protocol-agnostic definitions */ -std::string CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +std::string CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { return readString(conn.from); } -void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const std::string & str) +void CommonProto::Serialise::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const std::string & str) { conn.to << str; } -StorePath CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +StorePath CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { return store.parseStorePath(readString(conn.from)); } -void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const StorePath & storePath) +void CommonProto::Serialise::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const StorePath & storePath) { conn.to << store.printStorePath(storePath); } -ContentAddress CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +ContentAddress CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { return ContentAddress::parse(readString(conn.from)); } -void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const ContentAddress & ca) +void CommonProto::Serialise::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const ContentAddress & ca) { conn.to << renderContentAddress(ca); } -Realisation CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +Realisation CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { std::string rawInput = readString(conn.from); return Realisation::fromJSON( @@ -56,41 +56,41 @@ Realisation CommonProto::Serialise::read(const Store & store, Commo ); } -void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const Realisation & realisation) +void CommonProto::Serialise::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const Realisation & realisation) { conn.to << realisation.toJSON().dump(); } -DrvOutput CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +DrvOutput CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { return DrvOutput::parse(readString(conn.from)); } -void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput) +void CommonProto::Serialise::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput) { conn.to << drvOutput.to_string(); } -std::optional CommonProto::Serialise>::read(const Store & store, CommonProto::ReadConn conn) +std::optional CommonProto::Serialise>::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { auto s = readString(conn.from); return s == "" ? std::optional {} : store.parseStorePath(s); } -void CommonProto::Serialise>::write(const Store & store, CommonProto::WriteConn conn, const std::optional & storePathOpt) +void CommonProto::Serialise>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const std::optional & storePathOpt) { conn.to << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); } -std::optional CommonProto::Serialise>::read(const Store & store, CommonProto::ReadConn conn) +std::optional CommonProto::Serialise>::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { return ContentAddress::parseOpt(readString(conn.from)); } -void CommonProto::Serialise>::write(const Store & store, CommonProto::WriteConn conn, const std::optional & caOpt) +void CommonProto::Serialise>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const std::optional & caOpt) { conn.to << (caOpt ? renderContentAddress(*caOpt) : ""); } diff --git a/src/libstore/common-protocol.hh b/src/libstore/common-protocol.hh index f3f28972a..a878e84c9 100644 --- a/src/libstore/common-protocol.hh +++ b/src/libstore/common-protocol.hh @@ -5,7 +5,7 @@ namespace nix { -class Store; +struct StoreDirConfig; struct Source; // items being serialized @@ -48,7 +48,7 @@ struct CommonProto * infer the type instead of having to write it down explicitly. */ template - static void write(const Store & store, WriteConn conn, const T & t) + static void write(const StoreDirConfig & store, WriteConn conn, const T & t) { CommonProto::Serialise::write(store, conn, t); } @@ -57,8 +57,8 @@ struct CommonProto #define DECLARE_COMMON_SERIALISER(T) \ struct CommonProto::Serialise< T > \ { \ - static T read(const Store & store, CommonProto::ReadConn conn); \ - static void write(const Store & store, CommonProto::WriteConn conn, const T & str); \ + static T read(const StoreDirConfig & store, CommonProto::ReadConn conn); \ + static void write(const StoreDirConfig & store, CommonProto::WriteConn conn, const T & str); \ } template<> diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 1fecd1c97..239232c8e 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -11,7 +11,7 @@ namespace nix { -std::optional DerivationOutput::path(const Store & store, std::string_view drvName, OutputNameView outputName) const +std::optional DerivationOutput::path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const { return std::visit(overloaded { [](const DerivationOutput::InputAddressed & doi) -> std::optional { @@ -35,7 +35,7 @@ std::optional DerivationOutput::path(const Store & store, std::string } -StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, OutputNameView outputName) const +StorePath DerivationOutput::CAFixed::path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const { return store.makeFixedOutputPathFromCA( outputPathName(drvName, outputName), @@ -215,7 +215,7 @@ static StringSet parseStrings(std::istream & str, bool arePaths) static DerivationOutput parseDerivationOutput( - const Store & store, + const StoreDirConfig & store, std::string_view pathS, std::string_view hashAlgo, std::string_view hashS, const ExperimentalFeatureSettings & xpSettings) { @@ -262,7 +262,7 @@ static DerivationOutput parseDerivationOutput( } static DerivationOutput parseDerivationOutput( - const Store & store, std::istringstream & str, + const StoreDirConfig & store, std::istringstream & str, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings) { expect(str, ","); const auto pathS = parseString(str); @@ -291,7 +291,7 @@ enum struct DerivationATermVersion { }; static DerivedPathMap::ChildNode parseDerivedPathMapNode( - const Store & store, + const StoreDirConfig & store, std::istringstream & str, DerivationATermVersion version) { @@ -338,7 +338,7 @@ static DerivedPathMap::ChildNode parseDerivedPathMapNode( Derivation parseDerivation( - const Store & store, std::string && s, std::string_view name, + const StoreDirConfig & store, std::string && s, std::string_view name, const ExperimentalFeatureSettings & xpSettings) { Derivation drv; @@ -471,7 +471,7 @@ static void printUnquotedStrings(std::string & res, ForwardIterator i, ForwardIt } -static void unparseDerivedPathMapNode(const Store & store, std::string & s, const DerivedPathMap::ChildNode & node) +static void unparseDerivedPathMapNode(const StoreDirConfig & store, std::string & s, const DerivedPathMap::ChildNode & node) { s += ','; if (node.childMap.empty()) { @@ -512,7 +512,7 @@ static bool hasDynamicDrvDep(const Derivation & drv) } -std::string Derivation::unparse(const Store & store, bool maskOutputs, +std::string Derivation::unparse(const StoreDirConfig & store, bool maskOutputs, DerivedPathMap::ChildNode::Map * actualInputs) const { std::string s; @@ -846,7 +846,7 @@ std::map staticOutputHashes(Store & store, const Derivation & } -static DerivationOutput readDerivationOutput(Source & in, const Store & store) +static DerivationOutput readDerivationOutput(Source & in, const StoreDirConfig & store) { const auto pathS = readString(in); const auto hashAlgo = readString(in); @@ -863,7 +863,7 @@ StringSet BasicDerivation::outputNames() const return names; } -DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const +DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const StoreDirConfig & store) const { DerivationOutputsAndOptPaths outsAndOptPaths; for (auto & [outputName, output] : outputs) @@ -885,7 +885,7 @@ std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) } -Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name) +Source & readDerivation(Source & in, const StoreDirConfig & store, BasicDerivation & drv, std::string_view name) { drv.name = name; @@ -913,7 +913,7 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, } -void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv) +void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDerivation & drv) { out << drv.outputs.size(); for (auto & i : drv.outputs) { @@ -1154,7 +1154,7 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const const Hash impureOutputHash = hashString(htSHA256, "impure"); nlohmann::json DerivationOutput::toJSON( - const Store & store, std::string_view drvName, OutputNameView outputName) const + const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const { nlohmann::json res = nlohmann::json::object(); std::visit(overloaded { @@ -1181,7 +1181,7 @@ nlohmann::json DerivationOutput::toJSON( DerivationOutput DerivationOutput::fromJSON( - const Store & store, std::string_view drvName, OutputNameView outputName, + const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName, const nlohmann::json & _json, const ExperimentalFeatureSettings & xpSettings) { @@ -1250,7 +1250,7 @@ DerivationOutput DerivationOutput::fromJSON( } -nlohmann::json Derivation::toJSON(const Store & store) const +nlohmann::json Derivation::toJSON(const StoreDirConfig & store) const { nlohmann::json res = nlohmann::json::object(); @@ -1303,7 +1303,7 @@ nlohmann::json Derivation::toJSON(const Store & store) const Derivation Derivation::fromJSON( - const Store & store, + const StoreDirConfig & store, const nlohmann::json & json, const ExperimentalFeatureSettings & xpSettings) { diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index fa14e7536..219e8e7d7 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -17,7 +17,7 @@ namespace nix { -class Store; +struct StoreDirConfig; /* Abstract syntax of derivations. */ @@ -55,7 +55,7 @@ struct DerivationOutput * @param drvName The name of the derivation this is an output of, without the `.drv`. * @param outputName The name of this output. */ - StorePath path(const Store & store, std::string_view drvName, OutputNameView outputName) const; + StorePath path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const; GENERATE_CMP(CAFixed, me->ca); }; @@ -132,17 +132,17 @@ struct DerivationOutput * the safer interface provided by * BasicDerivation::outputsAndOptPaths */ - std::optional path(const Store & store, std::string_view drvName, OutputNameView outputName) const; + std::optional path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const; nlohmann::json toJSON( - const Store & store, + const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const; /** * @param xpSettings Stop-gap to avoid globals during unit tests. */ static DerivationOutput fromJSON( - const Store & store, + const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName, const nlohmann::json & json, @@ -304,7 +304,7 @@ struct BasicDerivation * augmented with knowledge of the Store paths they would be written * into. */ - DerivationOutputsAndOptPaths outputsAndOptPaths(const Store & store) const; + DerivationOutputsAndOptPaths outputsAndOptPaths(const StoreDirConfig & store) const; static std::string_view nameFromPath(const StorePath & storePath); @@ -318,6 +318,8 @@ struct BasicDerivation me->name); }; +class Store; + struct Derivation : BasicDerivation { /** @@ -328,7 +330,7 @@ struct Derivation : BasicDerivation /** * Print a derivation. */ - std::string unparse(const Store & store, bool maskOutputs, + std::string unparse(const StoreDirConfig & store, bool maskOutputs, DerivedPathMap::ChildNode::Map * actualInputs = nullptr) const; /** @@ -365,9 +367,9 @@ struct Derivation : BasicDerivation Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { } Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { } - nlohmann::json toJSON(const Store & store) const; + nlohmann::json toJSON(const StoreDirConfig & store) const; static Derivation fromJSON( - const Store & store, + const StoreDirConfig & store, const nlohmann::json & json, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); @@ -391,7 +393,7 @@ StorePath writeDerivation(Store & store, * Read a derivation from a file. */ Derivation parseDerivation( - const Store & store, + const StoreDirConfig & store, std::string && s, std::string_view name, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); @@ -493,8 +495,8 @@ extern Sync drvHashes; struct Source; struct Sink; -Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name); -void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv); +Source & readDerivation(Source & in, const StoreDirConfig & store, BasicDerivation & drv, std::string_view name); +void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDerivation & drv); /** * This creates an opaque and almost certainly unique string diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 47d784deb..3105dbc93 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -1,4 +1,5 @@ #include "derived-path.hh" +#include "derivations.hh" #include "store-api.hh" #include @@ -32,7 +33,7 @@ CMP(SingleDerivedPath, DerivedPathBuilt, outputs) #undef CMP #undef CMP_ONE -nlohmann::json DerivedPath::Opaque::toJSON(const Store & store) const +nlohmann::json DerivedPath::Opaque::toJSON(const StoreDirConfig & store) const { return store.printStorePath(path); } @@ -86,50 +87,50 @@ nlohmann::json DerivedPath::toJSON(Store & store) const }, raw()); } -std::string DerivedPath::Opaque::to_string(const Store & store) const +std::string DerivedPath::Opaque::to_string(const StoreDirConfig & store) const { return store.printStorePath(path); } -std::string SingleDerivedPath::Built::to_string(const Store & store) const +std::string SingleDerivedPath::Built::to_string(const StoreDirConfig & store) const { return drvPath->to_string(store) + "^" + output; } -std::string SingleDerivedPath::Built::to_string_legacy(const Store & store) const +std::string SingleDerivedPath::Built::to_string_legacy(const StoreDirConfig & store) const { return drvPath->to_string(store) + "!" + output; } -std::string DerivedPath::Built::to_string(const Store & store) const +std::string DerivedPath::Built::to_string(const StoreDirConfig & store) const { return drvPath->to_string(store) + '^' + outputs.to_string(); } -std::string DerivedPath::Built::to_string_legacy(const Store & store) const +std::string DerivedPath::Built::to_string_legacy(const StoreDirConfig & store) const { return drvPath->to_string_legacy(store) + "!" + outputs.to_string(); } -std::string SingleDerivedPath::to_string(const Store & store) const +std::string SingleDerivedPath::to_string(const StoreDirConfig & store) const { return std::visit( [&](const auto & req) { return req.to_string(store); }, raw()); } -std::string DerivedPath::to_string(const Store & store) const +std::string DerivedPath::to_string(const StoreDirConfig & store) const { return std::visit( [&](const auto & req) { return req.to_string(store); }, raw()); } -std::string SingleDerivedPath::to_string_legacy(const Store & store) const +std::string SingleDerivedPath::to_string_legacy(const StoreDirConfig & store) const { return std::visit(overloaded { [&](const SingleDerivedPath::Built & req) { return req.to_string_legacy(store); }, @@ -137,7 +138,7 @@ std::string SingleDerivedPath::to_string_legacy(const Store & store) const }, this->raw()); } -std::string DerivedPath::to_string_legacy(const Store & store) const +std::string DerivedPath::to_string_legacy(const StoreDirConfig & store) const { return std::visit(overloaded { [&](const DerivedPath::Built & req) { return req.to_string_legacy(store); }, @@ -146,7 +147,7 @@ std::string DerivedPath::to_string_legacy(const Store & store) const } -DerivedPath::Opaque DerivedPath::Opaque::parse(const Store & store, std::string_view s) +DerivedPath::Opaque DerivedPath::Opaque::parse(const StoreDirConfig & store, std::string_view s) { return {store.parseStorePath(s)}; } @@ -166,7 +167,7 @@ void drvRequireExperiment( } SingleDerivedPath::Built SingleDerivedPath::Built::parse( - const Store & store, ref drv, + const StoreDirConfig & store, ref drv, OutputNameView output, const ExperimentalFeatureSettings & xpSettings) { @@ -178,7 +179,7 @@ SingleDerivedPath::Built SingleDerivedPath::Built::parse( } DerivedPath::Built DerivedPath::Built::parse( - const Store & store, ref drv, + const StoreDirConfig & store, ref drv, OutputNameView outputsS, const ExperimentalFeatureSettings & xpSettings) { @@ -190,7 +191,7 @@ DerivedPath::Built DerivedPath::Built::parse( } static SingleDerivedPath parseWithSingle( - const Store & store, std::string_view s, std::string_view separator, + const StoreDirConfig & store, std::string_view s, std::string_view separator, const ExperimentalFeatureSettings & xpSettings) { size_t n = s.rfind(separator); @@ -207,7 +208,7 @@ static SingleDerivedPath parseWithSingle( } SingleDerivedPath SingleDerivedPath::parse( - const Store & store, + const StoreDirConfig & store, std::string_view s, const ExperimentalFeatureSettings & xpSettings) { @@ -215,7 +216,7 @@ SingleDerivedPath SingleDerivedPath::parse( } SingleDerivedPath SingleDerivedPath::parseLegacy( - const Store & store, + const StoreDirConfig & store, std::string_view s, const ExperimentalFeatureSettings & xpSettings) { @@ -223,7 +224,7 @@ SingleDerivedPath SingleDerivedPath::parseLegacy( } static DerivedPath parseWith( - const Store & store, std::string_view s, std::string_view separator, + const StoreDirConfig & store, std::string_view s, std::string_view separator, const ExperimentalFeatureSettings & xpSettings) { size_t n = s.rfind(separator); @@ -240,7 +241,7 @@ static DerivedPath parseWith( } DerivedPath DerivedPath::parse( - const Store & store, + const StoreDirConfig & store, std::string_view s, const ExperimentalFeatureSettings & xpSettings) { @@ -248,7 +249,7 @@ DerivedPath DerivedPath::parse( } DerivedPath DerivedPath::parseLegacy( - const Store & store, + const StoreDirConfig & store, std::string_view s, const ExperimentalFeatureSettings & xpSettings) { diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 4d7033df2..b12f9734a 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -12,6 +12,9 @@ namespace nix { +struct StoreDirConfig; + +// TODO stop needing this, `toJSON` below should be pure class Store; /** @@ -24,9 +27,9 @@ class Store; struct DerivedPathOpaque { StorePath path; - std::string to_string(const Store & store) const; - static DerivedPathOpaque parse(const Store & store, std::string_view); - nlohmann::json toJSON(const Store & store) const; + std::string to_string(const StoreDirConfig & store) const; + static DerivedPathOpaque parse(const StoreDirConfig & store, std::string_view); + nlohmann::json toJSON(const StoreDirConfig & store) const; GENERATE_CMP(DerivedPathOpaque, me->path); }; @@ -59,18 +62,18 @@ struct SingleDerivedPathBuilt { /** * Uses `^` as the separator */ - std::string to_string(const Store & store) const; + std::string to_string(const StoreDirConfig & store) const; /** * Uses `!` as the separator */ - std::string to_string_legacy(const Store & store) const; + std::string to_string_legacy(const StoreDirConfig & store) const; /** * The caller splits on the separator, so it works for both variants. * * @param xpSettings Stop-gap to avoid globals during unit tests. */ static SingleDerivedPathBuilt parse( - const Store & store, ref drvPath, + const StoreDirConfig & store, ref drvPath, OutputNameView outputs, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); nlohmann::json toJSON(Store & store) const; @@ -120,18 +123,18 @@ struct SingleDerivedPath : _SingleDerivedPathRaw { /** * Uses `^` as the separator */ - std::string to_string(const Store & store) const; + std::string to_string(const StoreDirConfig & store) const; /** * Uses `!` as the separator */ - std::string to_string_legacy(const Store & store) const; + std::string to_string_legacy(const StoreDirConfig & store) const; /** * Uses `^` as the separator * * @param xpSettings Stop-gap to avoid globals during unit tests. */ static SingleDerivedPath parse( - const Store & store, + const StoreDirConfig & store, std::string_view, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); /** @@ -140,7 +143,7 @@ struct SingleDerivedPath : _SingleDerivedPathRaw { * @param xpSettings Stop-gap to avoid globals during unit tests. */ static SingleDerivedPath parseLegacy( - const Store & store, + const StoreDirConfig & store, std::string_view, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); nlohmann::json toJSON(Store & store) const; @@ -182,18 +185,18 @@ struct DerivedPathBuilt { /** * Uses `^` as the separator */ - std::string to_string(const Store & store) const; + std::string to_string(const StoreDirConfig & store) const; /** * Uses `!` as the separator */ - std::string to_string_legacy(const Store & store) const; + std::string to_string_legacy(const StoreDirConfig & store) const; /** * The caller splits on the separator, so it works for both variants. * * @param xpSettings Stop-gap to avoid globals during unit tests. */ static DerivedPathBuilt parse( - const Store & store, ref, + const StoreDirConfig & store, ref, std::string_view, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); nlohmann::json toJSON(Store & store) const; @@ -242,18 +245,18 @@ struct DerivedPath : _DerivedPathRaw { /** * Uses `^` as the separator */ - std::string to_string(const Store & store) const; + std::string to_string(const StoreDirConfig & store) const; /** * Uses `!` as the separator */ - std::string to_string_legacy(const Store & store) const; + std::string to_string_legacy(const StoreDirConfig & store) const; /** * Uses `^` as the separator * * @param xpSettings Stop-gap to avoid globals during unit tests. */ static DerivedPath parse( - const Store & store, + const StoreDirConfig & store, std::string_view, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); /** @@ -262,7 +265,7 @@ struct DerivedPath : _DerivedPathRaw { * @param xpSettings Stop-gap to avoid globals during unit tests. */ static DerivedPath parseLegacy( - const Store & store, + const StoreDirConfig & store, std::string_view, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); diff --git a/src/libstore/length-prefixed-protocol-helper.hh b/src/libstore/length-prefixed-protocol-helper.hh index 4061b0cd6..0cf950a47 100644 --- a/src/libstore/length-prefixed-protocol-helper.hh +++ b/src/libstore/length-prefixed-protocol-helper.hh @@ -10,7 +10,7 @@ namespace nix { -class Store; +struct StoreDirConfig; /** * Reusable serialisers for serialization container types in a @@ -44,8 +44,8 @@ struct LengthPrefixedProtoHelper; #define LENGTH_PREFIXED_PROTO_HELPER(Inner, T) \ struct LengthPrefixedProtoHelper< Inner, T > \ { \ - static T read(const Store & store, typename Inner::ReadConn conn); \ - static void write(const Store & store, typename Inner::WriteConn conn, const T & str); \ + static T read(const StoreDirConfig & store, typename Inner::ReadConn conn); \ + static void write(const StoreDirConfig & store, typename Inner::WriteConn conn, const T & str); \ private: \ template using S = typename Inner::template Serialise; \ } @@ -67,7 +67,7 @@ LENGTH_PREFIXED_PROTO_HELPER(Inner, _X); template std::vector LengthPrefixedProtoHelper>::read( - const Store & store, typename Inner::ReadConn conn) + const StoreDirConfig & store, typename Inner::ReadConn conn) { std::vector resSet; auto size = readNum(conn.from); @@ -80,7 +80,7 @@ LengthPrefixedProtoHelper>::read( template void LengthPrefixedProtoHelper>::write( - const Store & store, typename Inner::WriteConn conn, const std::vector & resSet) + const StoreDirConfig & store, typename Inner::WriteConn conn, const std::vector & resSet) { conn.to << resSet.size(); for (auto & key : resSet) { @@ -91,7 +91,7 @@ LengthPrefixedProtoHelper>::write( template std::set LengthPrefixedProtoHelper>::read( - const Store & store, typename Inner::ReadConn conn) + const StoreDirConfig & store, typename Inner::ReadConn conn) { std::set resSet; auto size = readNum(conn.from); @@ -104,7 +104,7 @@ LengthPrefixedProtoHelper>::read( template void LengthPrefixedProtoHelper>::write( - const Store & store, typename Inner::WriteConn conn, const std::set & resSet) + const StoreDirConfig & store, typename Inner::WriteConn conn, const std::set & resSet) { conn.to << resSet.size(); for (auto & key : resSet) { @@ -115,7 +115,7 @@ LengthPrefixedProtoHelper>::write( template std::map LengthPrefixedProtoHelper>::read( - const Store & store, typename Inner::ReadConn conn) + const StoreDirConfig & store, typename Inner::ReadConn conn) { std::map resMap; auto size = readNum(conn.from); @@ -130,7 +130,7 @@ LengthPrefixedProtoHelper>::read( template void LengthPrefixedProtoHelper>::write( - const Store & store, typename Inner::WriteConn conn, const std::map & resMap) + const StoreDirConfig & store, typename Inner::WriteConn conn, const std::map & resMap) { conn.to << resMap.size(); for (auto & i : resMap) { @@ -142,7 +142,7 @@ LengthPrefixedProtoHelper>::write( template std::tuple LengthPrefixedProtoHelper>::read( - const Store & store, typename Inner::ReadConn conn) + const StoreDirConfig & store, typename Inner::ReadConn conn) { return std::tuple { S::read(store, conn)..., @@ -152,7 +152,7 @@ LengthPrefixedProtoHelper>::read( template void LengthPrefixedProtoHelper>::write( - const Store & store, typename Inner::WriteConn conn, const std::tuple & res) + const StoreDirConfig & store, typename Inner::WriteConn conn, const std::tuple & res) { std::apply([&](const Us &... args) { (S::write(store, conn, args), ...); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index a5e9426f8..e091683dc 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -4,6 +4,7 @@ #include "pathlocks.hh" #include "worker-protocol.hh" #include "derivations.hh" +#include "realisation.hh" #include "nar-info.hh" #include "references.hh" #include "callback.hh" diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 1035691c7..9f63fbbb5 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -4,6 +4,7 @@ #include "local-store.hh" #include "store-api.hh" #include "thread-pool.hh" +#include "realisation.hh" #include "topo-sort.hh" #include "callback.hh" #include "closure.hh" diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index af6837370..026e37647 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -5,7 +5,7 @@ namespace nix { -std::string StorePathWithOutputs::to_string(const Store & store) const +std::string StorePathWithOutputs::to_string(const StoreDirConfig & store) const { return outputs.empty() ? store.printStorePath(path) @@ -85,7 +85,7 @@ std::pair parsePathWithOutputs(std::string_view s) } -StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs) +StorePathWithOutputs parsePathWithOutputs(const StoreDirConfig & store, std::string_view pathWithOutputs) { auto [path, outputs] = parsePathWithOutputs(pathWithOutputs); return StorePathWithOutputs { store.parseStorePath(path), std::move(outputs) }; diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index 57e03252d..5f76a583a 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -6,6 +6,8 @@ namespace nix { +struct StoreDirConfig; + /** * This is a deprecated old type just for use by the old CLI, and older * versions of the RPC protocols. In new code don't use it; you want @@ -19,7 +21,7 @@ struct StorePathWithOutputs StorePath path; std::set outputs; - std::string to_string(const Store & store) const; + std::string to_string(const StoreDirConfig & store) const; DerivedPath toDerivedPath() const; @@ -32,14 +34,14 @@ std::vector toDerivedPaths(const std::vector) std::pair parsePathWithOutputs(std::string_view s); -class Store; - /** * Split a string specifying a derivation and a set of outputs * (/nix/store/hash-foo!out1,out2,...) into the derivation path * and the outputs. */ -StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs); +StorePathWithOutputs parsePathWithOutputs(const StoreDirConfig & store, std::string_view pathWithOutputs); + +class Store; StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs); diff --git a/src/libstore/serve-protocol-impl.hh b/src/libstore/serve-protocol-impl.hh index a3ce81026..6f3b177ac 100644 --- a/src/libstore/serve-protocol-impl.hh +++ b/src/libstore/serve-protocol-impl.hh @@ -16,11 +16,11 @@ namespace nix { /* protocol-agnostic templates */ #define SERVE_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \ - TEMPLATE T ServeProto::Serialise< T >::read(const Store & store, ServeProto::ReadConn conn) \ + TEMPLATE T ServeProto::Serialise< T >::read(const StoreDirConfig & store, ServeProto::ReadConn conn) \ { \ return LengthPrefixedProtoHelper::read(store, conn); \ } \ - TEMPLATE void ServeProto::Serialise< T >::write(const Store & store, ServeProto::WriteConn conn, const T & t) \ + TEMPLATE void ServeProto::Serialise< T >::write(const StoreDirConfig & store, ServeProto::WriteConn conn, const T & t) \ { \ LengthPrefixedProtoHelper::write(store, conn, t); \ } @@ -41,12 +41,12 @@ SERVE_USE_LENGTH_PREFIX_SERIALISER( template struct ServeProto::Serialise { - static T read(const Store & store, ServeProto::ReadConn conn) + static T read(const StoreDirConfig & store, ServeProto::ReadConn conn) { return CommonProto::Serialise::read(store, CommonProto::ReadConn { .from = conn.from }); } - static void write(const Store & store, ServeProto::WriteConn conn, const T & t) + static void write(const StoreDirConfig & store, ServeProto::WriteConn conn, const T & t) { CommonProto::Serialise::write(store, CommonProto::WriteConn { .to = conn.to }, diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc index 97a0ddf0e..e0ac80c4e 100644 --- a/src/libstore/serve-protocol.cc +++ b/src/libstore/serve-protocol.cc @@ -13,7 +13,7 @@ namespace nix { /* protocol-specific definitions */ -BuildResult ServeProto::Serialise::read(const Store & store, ServeProto::ReadConn conn) +BuildResult ServeProto::Serialise::read(const StoreDirConfig & store, ServeProto::ReadConn conn) { BuildResult status; status.status = (BuildResult::Status) readInt(conn.from); @@ -35,7 +35,7 @@ BuildResult ServeProto::Serialise::read(const Store & store, ServeP return status; } -void ServeProto::Serialise::write(const Store & store, ServeProto::WriteConn conn, const BuildResult & status) +void ServeProto::Serialise::write(const StoreDirConfig & store, ServeProto::WriteConn conn, const BuildResult & status) { conn.to << status.status diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index ba159f6e9..6e9d66e2d 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -13,7 +13,7 @@ namespace nix { #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) -class Store; +struct StoreDirConfig; struct Source; // items being serialised @@ -72,8 +72,8 @@ struct ServeProto // See `worker-protocol.hh` for a longer explanation. #if 0 { - static T read(const Store & store, ReadConn conn); - static void write(const Store & store, WriteConn conn, const T & t); + static T read(const StoreDirConfig & store, ReadConn conn); + static void write(const StoreDirConfig & store, WriteConn conn, const T & t); }; #endif @@ -82,7 +82,7 @@ struct ServeProto * infer the type instead of having to write it down explicitly. */ template - static void write(const Store & store, WriteConn conn, const T & t) + static void write(const StoreDirConfig & store, WriteConn conn, const T & t) { ServeProto::Serialise::write(store, conn, t); } @@ -135,8 +135,8 @@ inline std::ostream & operator << (std::ostream & s, ServeProto::Command op) #define DECLARE_SERVE_SERIALISER(T) \ struct ServeProto::Serialise< T > \ { \ - static T read(const Store & store, ServeProto::ReadConn conn); \ - static void write(const Store & store, ServeProto::WriteConn conn, const T & t); \ + static T read(const StoreDirConfig & store, ServeProto::ReadConn conn); \ + static void write(const StoreDirConfig & store, ServeProto::WriteConn conn, const T & t); \ }; template<> diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index a681bb6cf..e44376fa2 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1,6 +1,8 @@ #include "crypto.hh" #include "source-accessor.hh" #include "globals.hh" +#include "derived-path.hh" +#include "realisation.hh" #include "derivations.hh" #include "store-api.hh" #include "util.hh" diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index bee5ec16c..4342445ba 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -1,8 +1,6 @@ #pragma once ///@file -#include "nar-info.hh" -#include "realisation.hh" #include "path.hh" #include "derived-path.hh" #include "hash.hh" @@ -68,8 +66,13 @@ MakeError(SubstituterDisabled, Error); MakeError(InvalidStoreURI, Error); +struct Realisation; +struct RealisedPath; +struct DrvOutput; + struct BasicDerivation; struct Derivation; + struct SourceAccessor; class NarInfoDiskCache; class Store; @@ -811,7 +814,7 @@ void copyStorePath( */ std::map copyPaths( Store & srcStore, Store & dstStore, - const RealisedPath::Set &, + const std::set &, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); @@ -828,7 +831,7 @@ std::map copyPaths( */ void copyClosure( Store & srcStore, Store & dstStore, - const RealisedPath::Set & paths, + const std::set & paths, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); diff --git a/src/libstore/worker-protocol-impl.hh b/src/libstore/worker-protocol-impl.hh index c043588d6..026cc37bc 100644 --- a/src/libstore/worker-protocol-impl.hh +++ b/src/libstore/worker-protocol-impl.hh @@ -16,11 +16,11 @@ namespace nix { /* protocol-agnostic templates */ #define WORKER_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \ - TEMPLATE T WorkerProto::Serialise< T >::read(const Store & store, WorkerProto::ReadConn conn) \ + TEMPLATE T WorkerProto::Serialise< T >::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) \ { \ return LengthPrefixedProtoHelper::read(store, conn); \ } \ - TEMPLATE void WorkerProto::Serialise< T >::write(const Store & store, WorkerProto::WriteConn conn, const T & t) \ + TEMPLATE void WorkerProto::Serialise< T >::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const T & t) \ { \ LengthPrefixedProtoHelper::write(store, conn, t); \ } @@ -41,12 +41,12 @@ WORKER_USE_LENGTH_PREFIX_SERIALISER( template struct WorkerProto::Serialise { - static T read(const Store & store, WorkerProto::ReadConn conn) + static T read(const StoreDirConfig & store, WorkerProto::ReadConn conn) { return CommonProto::Serialise::read(store, CommonProto::ReadConn { .from = conn.from }); } - static void write(const Store & store, WorkerProto::WriteConn conn, const T & t) + static void write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const T & t) { CommonProto::Serialise::write(store, CommonProto::WriteConn { .to = conn.to }, diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index d618b9bd8..4edab7894 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -14,7 +14,7 @@ namespace nix { /* protocol-specific definitions */ -std::optional WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) +std::optional WorkerProto::Serialise>::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) { auto temp = readNum(conn.from); switch (temp) { @@ -29,7 +29,7 @@ std::optional WorkerProto::Serialise>::r } } -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional & optTrusted) +void WorkerProto::Serialise>::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const std::optional & optTrusted) { if (!optTrusted) conn.to << (uint8_t)0; @@ -48,7 +48,7 @@ void WorkerProto::Serialise>::write(const Store & sto } -DerivedPath WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) +DerivedPath WorkerProto::Serialise::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) { auto s = readString(conn.from); if (GET_PROTOCOL_MINOR(conn.version) >= 30) { @@ -58,7 +58,7 @@ DerivedPath WorkerProto::Serialise::read(const Store & store, Worke } } -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const DerivedPath & req) +void WorkerProto::Serialise::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const DerivedPath & req) { if (GET_PROTOCOL_MINOR(conn.version) >= 30) { conn.to << req.to_string_legacy(store); @@ -82,7 +82,7 @@ void WorkerProto::Serialise::write(const Store & store, WorkerProto } -KeyedBuildResult WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) +KeyedBuildResult WorkerProto::Serialise::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) { auto path = WorkerProto::Serialise::read(store, conn); auto br = WorkerProto::Serialise::read(store, conn); @@ -92,14 +92,14 @@ KeyedBuildResult WorkerProto::Serialise::read(const Store & st }; } -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const KeyedBuildResult & res) +void WorkerProto::Serialise::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const KeyedBuildResult & res) { WorkerProto::write(store, conn, res.path); WorkerProto::write(store, conn, static_cast(res)); } -BuildResult WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) +BuildResult WorkerProto::Serialise::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) { BuildResult res; res.status = (BuildResult::Status) readInt(conn.from); @@ -121,7 +121,7 @@ BuildResult WorkerProto::Serialise::read(const Store & store, Worke return res; } -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const BuildResult & res) +void WorkerProto::Serialise::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const BuildResult & res) { conn.to << res.status @@ -142,7 +142,7 @@ void WorkerProto::Serialise::write(const Store & store, WorkerProto } -ValidPathInfo WorkerProto::Serialise::read(const Store & store, ReadConn conn) +ValidPathInfo WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) { auto path = WorkerProto::Serialise::read(store, conn); return ValidPathInfo { @@ -151,14 +151,14 @@ ValidPathInfo WorkerProto::Serialise::read(const Store & store, R }; } -void WorkerProto::Serialise::write(const Store & store, WriteConn conn, const ValidPathInfo & pathInfo) +void WorkerProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const ValidPathInfo & pathInfo) { WorkerProto::write(store, conn, pathInfo.path); WorkerProto::write(store, conn, static_cast(pathInfo)); } -UnkeyedValidPathInfo WorkerProto::Serialise::read(const Store & store, ReadConn conn) +UnkeyedValidPathInfo WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) { auto deriver = readString(conn.from); auto narHash = Hash::parseAny(readString(conn.from), htSHA256); @@ -174,7 +174,7 @@ UnkeyedValidPathInfo WorkerProto::Serialise::read(const St return info; } -void WorkerProto::Serialise::write(const Store & store, WriteConn conn, const UnkeyedValidPathInfo & pathInfo) +void WorkerProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const UnkeyedValidPathInfo & pathInfo) { conn.to << (pathInfo.deriver ? store.printStorePath(*pathInfo.deriver) : "") diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index dcd54ad16..9b02aa2b5 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -24,7 +24,7 @@ namespace nix { #define STDERR_RESULT 0x52534c54 -class Store; +struct StoreDirConfig; struct Source; // items being serialised @@ -100,8 +100,8 @@ struct WorkerProto // This makes for a quicker debug cycle, as desired. #if 0 { - static T read(const Store & store, ReadConn conn); - static void write(const Store & store, WriteConn conn, const T & t); + static T read(const StoreDirConfig & store, ReadConn conn); + static void write(const StoreDirConfig & store, WriteConn conn, const T & t); }; #endif @@ -110,7 +110,7 @@ struct WorkerProto * infer the type instead of having to write it down explicitly. */ template - static void write(const Store & store, WriteConn conn, const T & t) + static void write(const StoreDirConfig & store, WriteConn conn, const T & t) { WorkerProto::Serialise::write(store, conn, t); } @@ -197,8 +197,8 @@ inline std::ostream & operator << (std::ostream & s, WorkerProto::Op op) #define DECLARE_WORKER_SERIALISER(T) \ struct WorkerProto::Serialise< T > \ { \ - static T read(const Store & store, WorkerProto::ReadConn conn); \ - static void write(const Store & store, WorkerProto::WriteConn conn, const T & t); \ + static T read(const StoreDirConfig & store, WorkerProto::ReadConn conn); \ + static void write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const T & t); \ }; template<> diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 60bc08146..c46095a14 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -13,6 +13,7 @@ #include "store-api.hh" #include "local-fs-store.hh" #include "globals.hh" +#include "realisation.hh" #include "derivations.hh" #include "util.hh" #include "shared.hh" diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc index 7f2bb93b6..b64af758f 100644 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -1,4 +1,5 @@ #include "shared.hh" +#include "realisation.hh" #include "store-api.hh" #include "legacy.hh" From e4cbdd26e0e6a2a5907dff8e60c3645f7d94423a Mon Sep 17 00:00:00 2001 From: Bouke van der Bijl Date: Mon, 13 Nov 2023 17:13:52 +0100 Subject: [PATCH 004/141] Add TODO comment for include try/catch --- src/libutil/config.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 17380b6d8..ab873b4a8 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -125,7 +125,9 @@ static void applyConfigInner(const std::string & contents, const std::string & p try { std::string includedContents = readFile(path); applyConfigInner(includedContents, p, parsedContents); - } catch (SysError &) { } + } catch (SysError &) { + // TODO: Do we actually want to ignore this? Or is it better to fail? + } } else if (!ignoreMissing) { throw Error("file '%1%' included from '%2%' not found", p, path); } From d6898cd58b1a685404ba6878c317e60be9473a9a Mon Sep 17 00:00:00 2001 From: Bouke van der Bijl Date: Mon, 13 Nov 2023 17:14:05 +0100 Subject: [PATCH 005/141] Move applyConfigFile to lambda inside libstore --- src/libstore/globals.cc | 11 +++++++++-- src/libutil/config.cc | 8 -------- src/libutil/config.hh | 6 ------ 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 9c25d9868..0aecd2b6a 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -111,7 +111,14 @@ Settings::Settings() void loadConfFile() { - globalConfig.applyConfigFile(settings.nixConfDir + "/nix.conf"); + auto applyConfigFile = [&](const Path & path) { + try { + std::string contents = readFile(path); + globalConfig.applyConfig(contents, path); + } catch (SysError &) { } + }; + + applyConfigFile(settings.nixConfDir + "/nix.conf"); /* We only want to send overrides to the daemon, i.e. stuff from ~/.nix/nix.conf or the command line. */ @@ -119,7 +126,7 @@ void loadConfFile() auto files = settings.nixUserConfFiles; for (auto file = files.rbegin(); file != files.rend(); file++) { - globalConfig.applyConfigFile(*file); + applyConfigFile(*file); } auto nixConfEnv = getEnv("NIX_CONFIG"); diff --git a/src/libutil/config.cc b/src/libutil/config.cc index ab873b4a8..ad16c86bd 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -165,14 +165,6 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string set(name, value); } -void AbstractConfig::applyConfigFile(const Path & path) -{ - try { - std::string contents = readFile(path); - applyConfig(contents, path); - } catch (SysError &) { } -} - void Config::resetOverridden() { for (auto & s : _settings) diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 38c3ce0c4..d49eb602d 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -82,12 +82,6 @@ public: */ void applyConfig(const std::string & contents, const std::string & path = ""); - /** - * Applies a nix configuration file - * - path: the location of the config file to apply - */ - void applyConfigFile(const Path & path); - /** * Resets the `overridden` flag of all Settings */ From 4e790efade0c3073292ff73be44351f29badd935 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 20 Nov 2023 13:38:52 +0100 Subject: [PATCH 006/141] Use boost::container::small_vector in place of VLAs --- boehmgc-traceable_allocator-public.diff | 12 +++++++ flake.nix | 3 ++ src/libexpr/eval.cc | 13 +++++--- src/libexpr/gc-small-vector.hh | 42 +++++++++++++++++++++++++ src/libexpr/primops.cc | 9 +++--- 5 files changed, 70 insertions(+), 9 deletions(-) create mode 100644 boehmgc-traceable_allocator-public.diff create mode 100644 src/libexpr/gc-small-vector.hh diff --git a/boehmgc-traceable_allocator-public.diff b/boehmgc-traceable_allocator-public.diff new file mode 100644 index 000000000..903c707a6 --- /dev/null +++ b/boehmgc-traceable_allocator-public.diff @@ -0,0 +1,12 @@ +diff --git a/include/gc_allocator.h b/include/gc_allocator.h +index 597c7f13..587286be 100644 +--- a/include/gc_allocator.h ++++ b/include/gc_allocator.h +@@ -312,6 +312,7 @@ public: + + template<> + class traceable_allocator { ++public: + typedef size_t size_type; + typedef ptrdiff_t difference_type; + typedef void* pointer; diff --git a/flake.nix b/flake.nix index 9030a74f7..570a099ab 100644 --- a/flake.nix +++ b/flake.nix @@ -230,6 +230,9 @@ }).overrideAttrs(o: { patches = (o.patches or []) ++ [ ./boehmgc-coroutine-sp-fallback.diff + + # https://github.com/ivmai/bdwgc/pull/586 + ./boehmgc-traceable_allocator-public.diff ]; }) ) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 46a49c891..90f04d40a 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -16,6 +16,7 @@ #include "fs-input-accessor.hh" #include "memory-input-accessor.hh" #include "signals.hh" +#include "gc-small-vector.hh" #include #include @@ -31,6 +32,7 @@ #include #include +#include #if HAVE_BOEHMGC @@ -1709,7 +1711,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & /* We have all the arguments, so call the primop with the previous and new arguments. */ - Value * vArgs[arity]; + Value * vArgs[maxPrimOpArity]; auto n = argsDone; for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->primOpApp.left) vArgs[--n] = arg->primOpApp.right; @@ -1772,11 +1774,11 @@ void ExprCall::eval(EvalState & state, Env & env, Value & v) // 4: about 60 // 5: under 10 // This excluded attrset lambdas (`{...}:`). Contributions of mixed lambdas appears insignificant at ~150 total. - Value * vArgs[args.size()]; + SmallValueVector<4> vArgs(args.size()); for (size_t i = 0; i < args.size(); ++i) vArgs[i] = args[i]->maybeThunk(state, env); - state.callFunction(vFun, args.size(), vArgs, v, pos); + state.callFunction(vFun, args.size(), vArgs.data(), v, pos); } @@ -2015,8 +2017,9 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) return result; }; - Value values[es->size()]; - Value * vTmpP = values; + // List of returned strings. References to these Values must NOT be persisted. + SmallTemporaryValueVector values(es->size()); + Value * vTmpP = values.data(); for (auto & [i_pos, i] : *es) { Value & vTmp = *vTmpP++; diff --git a/src/libexpr/gc-small-vector.hh b/src/libexpr/gc-small-vector.hh new file mode 100644 index 000000000..7f4f08fc7 --- /dev/null +++ b/src/libexpr/gc-small-vector.hh @@ -0,0 +1,42 @@ +#pragma once + +#include + +#if HAVE_BOEHMGC + +#include +#include +#include + +#endif + +namespace nix { + +struct Value; + +/** + * A GC compatible vector that may used a reserved portion of `nItems` on the stack instead of allocating on the heap. + */ +#if HAVE_BOEHMGC +template +using SmallVector = boost::container::small_vector>; +#else +template +using SmallVector = boost::container::small_vector; +#endif + +/** + * A vector of value pointers. See `SmallVector`. + */ +template +using SmallValueVector = SmallVector; + +/** + * A vector of values that must not be referenced after the vector is destroyed. + * + * See also `SmallValueVector`. + */ +template +using SmallTemporaryValueVector = SmallVector; + +} \ No newline at end of file diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index a8d44d8b7..54a5da817 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -4,6 +4,7 @@ #include "eval-inline.hh" #include "eval.hh" #include "eval-settings.hh" +#include "gc-small-vector.hh" #include "globals.hh" #include "json-to-value.hh" #include "names.hh" @@ -2729,7 +2730,7 @@ static void prim_catAttrs(EvalState & state, const PosIdx pos, Value * * args, V auto attrName = state.symbols.create(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.catAttrs")); state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.catAttrs"); - Value * res[args[1]->listSize()]; + SmallValueVector res(args[1]->listSize()); size_t found = 0; for (auto v2 : args[1]->listItems()) { @@ -3064,8 +3065,7 @@ static void prim_filter(EvalState & state, const PosIdx pos, Value * * args, Val state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filter"); - // FIXME: putting this on the stack is risky. - Value * vs[args[1]->listSize()]; + SmallValueVector vs(args[1]->listSize()); size_t k = 0; bool same = true; @@ -3454,7 +3454,8 @@ static void prim_concatMap(EvalState & state, const PosIdx pos, Value * * args, state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.concatMap"); auto nrLists = args[1]->listSize(); - Value lists[nrLists]; + // List of returned lists before concatenation. References to these Values must NOT be persisted. + SmallTemporaryValueVector lists(nrLists); size_t len = 0; for (unsigned int n = 0; n < nrLists; ++n) { From 20cd5eb2b3668f5b95b6f020e1c258011c18ea33 Mon Sep 17 00:00:00 2001 From: Alois Wohlschlager Date: Mon, 27 Nov 2023 19:12:15 +0100 Subject: [PATCH 007/141] nix repl: Only hide the progress bar while waiting for user input In commit 0d2163c6dcf03463fa91ec6d0d96c928ad907366, the progress bar was hidden in nix repl because of a regression that caused it to interfere with user input. Several users like(d) seeing the progress bar in the repl during builds. Only hiding it while waiting for user input gives us the best of both worlds, so do just that. --- src/libcmd/repl.cc | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index bf5643a5c..0986296ad 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -43,7 +43,6 @@ extern "C" { #include "finally.hh" #include "markdown.hh" #include "local-fs-store.hh" -#include "progress-bar.hh" #include "print.hh" #if HAVE_BOEHMGC @@ -262,13 +261,11 @@ void NixRepl::mainLoop() rl_set_list_possib_func(listPossibleCallback); #endif - /* Stop the progress bar because it interferes with the display of - the repl. */ - stopProgressBar(); - std::string input; while (true) { + // Hide the progress bar while waiting for user input, so that it won't interfere. + logger->pause(); // When continuing input from previous lines, don't print a prompt, just align to the same // number of chars as the prompt. if (!getLine(input, input.empty() ? "nix-repl> " : " ")) { @@ -278,6 +275,7 @@ void NixRepl::mainLoop() logger->cout(""); break; } + logger->resume(); try { if (!removeWhitespace(input).empty() && !processLine(input)) return; } catch (ParseError & e) { From 6d1605818c12461edc9f4ee17a7929fdc8fe916c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 27 Nov 2023 19:46:05 +0100 Subject: [PATCH 008/141] Rename `nix doctor` to `nix config check` Fix #7672 --- doc/manual/src/contributing/cli-guideline.md | 2 +- src/nix/{doctor.cc => config-check.cc} | 4 ++-- src/nix/main.cc | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) rename src/nix/{doctor.cc => config-check.cc} (97%) diff --git a/doc/manual/src/contributing/cli-guideline.md b/doc/manual/src/contributing/cli-guideline.md index e53d2d178..8dbac45b0 100644 --- a/doc/manual/src/contributing/cli-guideline.md +++ b/doc/manual/src/contributing/cli-guideline.md @@ -87,7 +87,7 @@ impacted the most by bad user experience. and [aligning of text](#text-alignment). - [Autocomplete](#shell-completion) of options. - Examples of such commands: `nix doctor`, `nix edit`, `nix eval`, ... + Examples of such commands: `nix edit`, `nix eval`, ... - **Utility and scripting commands** diff --git a/src/nix/doctor.cc b/src/nix/config-check.cc similarity index 97% rename from src/nix/doctor.cc rename to src/nix/config-check.cc index 59f9e3e5d..410feca2f 100644 --- a/src/nix/doctor.cc +++ b/src/nix/config-check.cc @@ -38,7 +38,7 @@ void checkInfo(const std::string & msg) { } -struct CmdDoctor : StoreCommand +struct CmdConfigCheck : StoreCommand { bool success = true; @@ -152,4 +152,4 @@ struct CmdDoctor : StoreCommand } }; -static auto rCmdDoctor = registerCommand("doctor"); +static auto rCmdConfigCheck = registerCommand2({ "config", "check" }); diff --git a/src/nix/main.cc b/src/nix/main.cc index 2a6c2f478..d715d0400 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -139,6 +139,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs {"to-base32", {"hash", "to-base32"}}, {"to-base64", {"hash", "to-base64"}}, {"verify", {"store", "verify"}}, + {"doctor", {"config", "check"}}, }; bool aliasUsed = false; From 52e0911302b20336c1600b60a98894423e110d7d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 25 Nov 2023 00:33:21 -0500 Subject: [PATCH 009/141] Use `buildprefix` in a few more places `installcheck` doesn't yet work, but the rest of the build can now happen mostly inside a separate build directory. Progress on #9342 Co-authored-by: Valentin Gagarin --- Makefile | 6 ++++-- doc/manual/src/contributing/hacking.md | 25 +++++++++++++++++++++++++ mk/build-dir.mk | 10 ++++++++++ mk/install-dirs.mk | 11 +++++++++++ mk/lib.mk | 25 +++---------------------- mk/templates.mk | 8 ++++---- src/libcmd/local.mk | 2 +- src/libexpr/local.mk | 4 ++-- src/libmain/local.mk | 2 +- src/libstore/local.mk | 2 +- tests/functional/local.mk | 4 ++-- 11 files changed, 64 insertions(+), 35 deletions(-) create mode 100644 mk/build-dir.mk create mode 100644 mk/install-dirs.mk diff --git a/Makefile b/Makefile index 4f4ac0c6e..77974074c 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,7 @@ --include Makefile.config -clean-files += Makefile.config +include mk/build-dir.mk + +-include $(buildprefix)Makefile.config +clean-files += $(buildprefix)Makefile.config ifeq ($(ENABLE_BUILD), yes) makefiles = \ diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 3291d5a20..0a95334f7 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -146,6 +146,31 @@ $ nix build .#packages.aarch64-linux.default Cross-compiled builds are available for ARMv6 (`armv6l-linux`) and ARMv7 (`armv7l-linux`). Add more [system types](#system-type) to `crossSystems` in `flake.nix` to bootstrap Nix on unsupported platforms. +### Building for multiple platforms at once + +It is useful to perform multiple cross and native builds on the same source tree, +for example to ensure that better support for one platform doesn't break the build for another. +In order to facilitate this, Nix has some support for being built out of tree – that is, placing build artefacts in a different directory than the source code: + +1. Create a directory for the build, e.g. + + ```bash + mkdir build + ``` + +2. Run the configure script from that directory, e.g. + + ```bash + cd build + ../configure + ``` + +3. Run make from the source directory, but with the build directory specified, e.g. + + ```bash + make builddir=build + ``` + ## System type Nix uses a string with he following format to identify the *system type* or *platform* it runs on: diff --git a/mk/build-dir.mk b/mk/build-dir.mk new file mode 100644 index 000000000..02f4cae60 --- /dev/null +++ b/mk/build-dir.mk @@ -0,0 +1,10 @@ +# Initialise support for build directories. +builddir ?= + +ifdef builddir + buildprefix = $(builddir)/ + buildprefixrel = $(builddir) +else + buildprefix = + buildprefixrel = . +endif diff --git a/mk/install-dirs.mk b/mk/install-dirs.mk new file mode 100644 index 000000000..732b0d6fc --- /dev/null +++ b/mk/install-dirs.mk @@ -0,0 +1,11 @@ +# Default installation paths. +prefix ?= /usr/local +libdir ?= $(prefix)/lib +bindir ?= $(prefix)/bin +libexecdir ?= $(prefix)/libexec +datadir ?= $(prefix)/share +localstatedir ?= $(prefix)/var +sysconfdir ?= $(prefix)/etc +mandir ?= $(prefix)/share/man + +DESTDIR ?= diff --git a/mk/lib.mk b/mk/lib.mk index 49abe9862..3d503364f 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -43,27 +43,6 @@ define newline endef -# Default installation paths. -prefix ?= /usr/local -libdir ?= $(prefix)/lib -bindir ?= $(prefix)/bin -libexecdir ?= $(prefix)/libexec -datadir ?= $(prefix)/share -localstatedir ?= $(prefix)/var -sysconfdir ?= $(prefix)/etc -mandir ?= $(prefix)/share/man - - -# Initialise support for build directories. -builddir ?= - -ifdef builddir - buildprefix = $(builddir)/ -else - buildprefix = -endif - - # Pass -fPIC if we're building dynamic libraries. BUILD_SHARED_LIBS ?= 1 @@ -94,6 +73,8 @@ ifeq ($(BUILD_DEBUG), 1) endif +include mk/build-dir.mk +include mk/install-dirs.mk include mk/functions.mk include mk/tracing.mk include mk/clean.mk @@ -112,7 +93,7 @@ define include-sub-makefile include $(1) endef -$(foreach mf, $(makefiles), $(eval $(call include-sub-makefile, $(mf)))) +$(foreach mf, $(makefiles), $(eval $(call include-sub-makefile,$(mf)))) # Instantiate stuff. diff --git a/mk/templates.mk b/mk/templates.mk index c7ac7afbf..866bdc17f 100644 --- a/mk/templates.mk +++ b/mk/templates.mk @@ -10,10 +10,10 @@ endef ifneq ($(MAKECMDGOALS), clean) -%.h: %.h.in - $(trace-gen) rm -f $@ && ./config.status --quiet --header=$@ +$(buildprefix)%.h: %.h.in + $(trace-gen) rm -f $@ && cd $(buildprefixrel) && ./config.status --quiet --header=$(@:$(buildprefix)%=%) -%: %.in - $(trace-gen) rm -f $@ && ./config.status --quiet --file=$@ +$(buildprefix)%: %.in + $(trace-gen) rm -f $@ && cd $(buildprefixrel) && ./config.status --quiet --file=$(@:$(buildprefix)%=%) endif diff --git a/src/libcmd/local.mk b/src/libcmd/local.mk index 541a7d2ba..afd35af08 100644 --- a/src/libcmd/local.mk +++ b/src/libcmd/local.mk @@ -12,4 +12,4 @@ libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) -pthread libcmd_LIBS = libstore libutil libexpr libmain libfetchers -$(eval $(call install-file-in, $(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644)) +$(eval $(call install-file-in, $(buildprefix)$(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644)) diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 18d1bc95c..c07a18bb5 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -36,7 +36,7 @@ $(d)/lexer-tab.cc $(d)/lexer-tab.hh: $(d)/lexer.l clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh -$(eval $(call install-file-in, $(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644)) +$(eval $(call install-file-in, $(buildprefix)$(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644)) $(foreach i, $(wildcard src/libexpr/value/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix/value, 0644))) @@ -47,4 +47,4 @@ $(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/flake/call-flake.nix.gen.hh -src/libexpr/primops/fromTOML.o: ERROR_SWITCH_ENUM = +$(buildprefix)src/libexpr/primops/fromTOML.o: ERROR_SWITCH_ENUM = diff --git a/src/libmain/local.mk b/src/libmain/local.mk index 99da95e27..5c7061863 100644 --- a/src/libmain/local.mk +++ b/src/libmain/local.mk @@ -14,4 +14,4 @@ libmain_LIBS = libstore libutil libmain_ALLOW_UNDEFINED = 1 -$(eval $(call install-file-in, $(d)/nix-main.pc, $(libdir)/pkgconfig, 0644)) +$(eval $(call install-file-in, $(buildprefix)$(d)/nix-main.pc, $(libdir)/pkgconfig, 0644)) diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 0be0bf310..68ccdc409 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -59,7 +59,7 @@ $(d)/build.cc: clean-files += $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh -$(eval $(call install-file-in, $(d)/nix-store.pc, $(libdir)/pkgconfig, 0644)) +$(eval $(call install-file-in, $(buildprefix)$(d)/nix-store.pc, $(libdir)/pkgconfig, 0644)) $(foreach i, $(wildcard src/libstore/builtins/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix/builtins, 0644))) diff --git a/tests/functional/local.mk b/tests/functional/local.mk index 8d584142a..10b399d75 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -140,9 +140,9 @@ ifeq ($(ENABLE_BUILD), yes) endif $(d)/test-libstoreconsumer.sh.test $(d)/test-libstoreconsumer.sh.test-debug: \ - $(d)/test-libstoreconsumer/test-libstoreconsumer + $(buildprefix)$(d)/test-libstoreconsumer/test-libstoreconsumer $(d)/plugins.sh.test $(d)/plugins.sh.test-debug: \ - $(d)/plugins/libplugintest.$(SO_EXT) + $(buildprefix)$(d)/plugins/libplugintest.$(SO_EXT) install-tests += $(foreach x, $(nix_tests), $(d)/$(x)) From 02bd821f2e71372d31bbe6700bd68086cc2ee70a Mon Sep 17 00:00:00 2001 From: Alex Ameen Date: Wed, 29 Nov 2023 19:26:39 -0600 Subject: [PATCH 010/141] fix: `nlohmann::adl_serializer` for `std::optional` (#9147) This allows templates such as `NLOHMANN_DEFINE_TYPE_*` templates and other generators with things like `std::vector>`. Co-authored-by: John Ericson --- src/libutil/json-utils.hh | 19 ++++++++--- src/libutil/tests/json-utils.cc | 58 +++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 5 deletions(-) create mode 100644 src/libutil/tests/json-utils.cc diff --git a/src/libutil/json-utils.hh b/src/libutil/json-utils.hh index 77c63595c..06dd80cf7 100644 --- a/src/libutil/json-utils.hh +++ b/src/libutil/json-utils.hh @@ -78,20 +78,29 @@ namespace nlohmann { */ template struct adl_serializer> { - static std::optional from_json(const json & json) { + /** + * @brief Convert a JSON type to an `optional` treating + * `null` as `std::nullopt`. + */ + static void from_json(const json & json, std::optional & t) { static_assert( nix::json_avoids_null::value, "null is already in use for underlying type's JSON"); - return json.is_null() + t = json.is_null() ? std::nullopt - : std::optional { adl_serializer::from_json(json) }; + : std::make_optional(json.template get()); } - static void to_json(json & json, std::optional t) { + + /** + * @brief Convert an optional type to a JSON type treating `std::nullopt` + * as `null`. + */ + static void to_json(json & json, const std::optional & t) { static_assert( nix::json_avoids_null::value, "null is already in use for underlying type's JSON"); if (t) - adl_serializer::to_json(json, *t); + json = *t; else json = nullptr; } diff --git a/src/libutil/tests/json-utils.cc b/src/libutil/tests/json-utils.cc new file mode 100644 index 000000000..f0ce15c93 --- /dev/null +++ b/src/libutil/tests/json-utils.cc @@ -0,0 +1,58 @@ +#include +#include + +#include + +#include "json-utils.hh" + +namespace nix { + +/* Test `to_json` and `from_json` with `std::optional` types. + * We are specifically interested in whether we can _nest_ optionals in STL + * containers so we that we can leverage existing adl_serializer templates. */ + +TEST(to_json, optionalInt) { + std::optional val = std::make_optional(420); + ASSERT_EQ(nlohmann::json(val), nlohmann::json(420)); + val = std::nullopt; + ASSERT_EQ(nlohmann::json(val), nlohmann::json(nullptr)); +} + +TEST(to_json, vectorOfOptionalInts) { + std::vector> vals = { + std::make_optional(420), + std::nullopt, + }; + ASSERT_EQ(nlohmann::json(vals), nlohmann::json::parse("[420,null]")); +} + +TEST(to_json, optionalVectorOfInts) { + std::optional> val = std::make_optional(std::vector { + -420, + 420, + }); + ASSERT_EQ(nlohmann::json(val), nlohmann::json::parse("[-420,420]")); + val = std::nullopt; + ASSERT_EQ(nlohmann::json(val), nlohmann::json(nullptr)); +} + +TEST(from_json, optionalInt) { + nlohmann::json json = 420; + std::optional val = json; + ASSERT_TRUE(val.has_value()); + ASSERT_EQ(*val, 420); + json = nullptr; + json.get_to(val); + ASSERT_FALSE(val.has_value()); +} + +TEST(from_json, vectorOfOptionalInts) { + nlohmann::json json = { 420, nullptr }; + std::vector> vals = json; + ASSERT_EQ(vals.size(), 2); + ASSERT_TRUE(vals.at(0).has_value()); + ASSERT_EQ(*vals.at(0), 420); + ASSERT_FALSE(vals.at(1).has_value()); +} + +} /* namespace nix */ From a7115a47ef0d83ea81b494f6bc5b11d8286e0672 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 29 Nov 2023 21:03:56 -0500 Subject: [PATCH 011/141] Improve ACL clearing support (fixing FreeBSD build) The problem was that f880469173061a07f0b2a24734932c5a9ad633c6 forgot that the `#include ` was guarded by an `#ifdef __linux__`. However, the build failure was only on FreeBSD --- turns out other platforms have this header too! The fix therefore uses a new configure check so we properly clear ACLs on more platforms. --- configure.ac | 2 ++ src/libstore/local-store.cc | 1 - src/libstore/posix-fs-canonicalise.cc | 6 ++++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index 71e93feaa..f8b937eb5 100644 --- a/configure.ac +++ b/configure.ac @@ -282,6 +282,8 @@ case "$host_os" in esac AC_SUBST(HAVE_SECCOMP, [$have_seccomp]) +# Optional dependencies for better normalizing file system data +AC_CHECK_HEADERS[sys/xattr.h] # Look for aws-cpp-sdk-s3. AC_LANG_PUSH(C++) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 4ff75f528..9ed061b01 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -34,7 +34,6 @@ #include #include #include -#include #endif #ifdef __CYGWIN__ diff --git a/src/libstore/posix-fs-canonicalise.cc b/src/libstore/posix-fs-canonicalise.cc index cc3ab0b74..f38fa8369 100644 --- a/src/libstore/posix-fs-canonicalise.cc +++ b/src/libstore/posix-fs-canonicalise.cc @@ -1,4 +1,6 @@ -#include +#if HAVE_SYS_XATTR_H +# include +#endif #include "posix-fs-canonicalise.hh" #include "file-system.hh" @@ -76,7 +78,7 @@ static void canonicalisePathMetaData_( if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))) throw Error("file '%1%' has an unsupported type", path); -#if __linux__ +#ifdef HAVE_SYS_XATTR_H /* Remove extended attributes / ACLs. */ ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0); From d536c57e878a04f795c1ef8ee3232a47035da2cf Mon Sep 17 00:00:00 2001 From: Federico Pellegrin Date: Tue, 17 Aug 2021 04:26:41 +0200 Subject: [PATCH 012/141] Docs build: depend on locally built nix executable and not installed one Previously many of the documentation targets were depending on `$(bindir)/nix` which is the installed version. This meant that its install rules would be triggered (which in chain would also trigger the install of libraries, as reported in #5140). Therefore a build of the documentation without an installation would not be possible (which apart from doing unwanted operations it may also generate permission problems for example). The fix makes the rules depend on `$(nix_PATH)` instead, which is the executable in the build tree. --- Makefile | 11 ++++++++--- doc/manual/local.mk | 35 ++++++++++++++++++++--------------- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index 4f4ac0c6e..957920686 100644 --- a/Makefile +++ b/Makefile @@ -19,9 +19,7 @@ makefiles = \ misc/zsh/local.mk \ misc/systemd/local.mk \ misc/launchd/local.mk \ - misc/upstart/local.mk \ - doc/manual/local.mk \ - doc/internal-api/local.mk + misc/upstart/local.mk endif ifeq ($(ENABLE_BUILD)_$(ENABLE_TESTS), yes_yes) @@ -55,4 +53,11 @@ endif include mk/lib.mk +# Must be included after `mk/lib.mk` so rules refer to variables defined +# by the library. Rules are not "lazy" like variables, unfortunately. +ifeq ($(ENABLE_BUILD), yes) +$(eval $(call include-sub-makefile, doc/manual/local.mk)) +$(eval $(call include-sub-makefile, doc/internal-api/local.mk)) +endif + GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++2a -I src diff --git a/doc/manual/local.mk b/doc/manual/local.mk index d568681d4..fa9db9f02 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -1,5 +1,10 @@ ifeq ($(doc_generate),yes) +# The version of Nix used to generate the doc. Can also be +# `$(nix_INSTALL_PATH)` or just `nix` (to grap ambient from the `PATH`), +# if one prefers. +doc_nix = $(nix_PATH) + MANUAL_SRCS := \ $(call rwildcard, $(d)/src, *.md) \ $(call rwildcard, $(d)/src, */*.md) @@ -32,7 +37,7 @@ dummy-env = env -i \ NIX_STATE_DIR=/dummy \ NIX_CONFIG='cores = 0' -nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix=doc/manual --store dummy:// --impure --raw +nix-eval = $(dummy-env) $(doc_nix) eval --experimental-features nix-command -I nix=doc/manual --store dummy:// --impure --raw # re-implement mdBook's include directive to make it usable for terminal output and for proper @docroot@ substitution define process-includes @@ -96,52 +101,52 @@ $(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src @cp $< $@ @$(call process-includes,$@,$@) -$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(d)/generate-settings.nix $(d)/generate-store-info.nix $(bindir)/nix +$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(d)/generate-settings.nix $(d)/generate-store-info.nix $(doc_nix) @rm -rf $@ $@.tmp $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix true (builtins.readFile $<)' @mv $@.tmp $@ -$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/generate-settings.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix +$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/generate-settings.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(doc_nix) @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-settings.nix { prefix = "conf"; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; @mv $@.tmp $@ -$(d)/nix.json: $(bindir)/nix - $(trace-gen) $(dummy-env) $(bindir)/nix __dump-cli > $@.tmp +$(d)/nix.json: $(doc_nix) + $(trace-gen) $(dummy-env) $(doc_nix) __dump-cli > $@.tmp @mv $@.tmp $@ -$(d)/conf-file.json: $(bindir)/nix - $(trace-gen) $(dummy-env) $(bindir)/nix config show --json --experimental-features nix-command > $@.tmp +$(d)/conf-file.json: $(doc_nix) + $(trace-gen) $(dummy-env) $(doc_nix) config show --json --experimental-features nix-command > $@.tmp @mv $@.tmp $@ -$(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(bindir)/nix +$(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(doc_nix) @rm -rf $@ $@.tmp $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features.nix (builtins.fromJSON (builtins.readFile $<))' @mv $@.tmp $@ -$(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(bindir)/nix +$(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(doc_nix) @rm -rf $@ $@.tmp $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features-shortlist.nix (builtins.fromJSON (builtins.readFile $<))' @mv $@.tmp $@ -$(d)/xp-features.json: $(bindir)/nix - $(trace-gen) $(dummy-env) $(bindir)/nix __dump-xp-features > $@.tmp +$(d)/xp-features.json: $(doc_nix) + $(trace-gen) $(dummy-env) $(doc_nix) __dump-xp-features > $@.tmp @mv $@.tmp $@ -$(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix +$(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(doc_nix) @cat doc/manual/src/language/builtins-prefix.md > $@.tmp $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<)).builtins' >> $@.tmp; @cat doc/manual/src/language/builtins-suffix.md >> $@.tmp @mv $@.tmp $@ -$(d)/src/language/builtin-constants.md: $(d)/language.json $(d)/generate-builtin-constants.nix $(d)/src/language/builtin-constants-prefix.md $(bindir)/nix +$(d)/src/language/builtin-constants.md: $(d)/language.json $(d)/generate-builtin-constants.nix $(d)/src/language/builtin-constants-prefix.md $(doc_nix) @cat doc/manual/src/language/builtin-constants-prefix.md > $@.tmp $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtin-constants.nix (builtins.fromJSON (builtins.readFile $<)).constants' >> $@.tmp; @cat doc/manual/src/language/builtin-constants-suffix.md >> $@.tmp @mv $@.tmp $@ -$(d)/language.json: $(bindir)/nix - $(trace-gen) $(dummy-env) $(bindir)/nix __dump-language > $@.tmp +$(d)/language.json: $(doc_nix) + $(trace-gen) $(dummy-env) $(doc_nix) __dump-language > $@.tmp @mv $@.tmp $@ # Generate "Upcoming release" notes (or clear it and remove from menu) From 743232bf04d8ade18cfa2c791ed466ce48519878 Mon Sep 17 00:00:00 2001 From: Greg Pfeil Date: Thu, 30 Nov 2023 00:17:25 -0700 Subject: [PATCH 013/141] =?UTF-8?q?Don=E2=80=99t=20use=20`execvp`=20when?= =?UTF-8?q?=20we=20know=20the=20path?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/nix/develop.cc | 2 +- src/nix/fmt.cc | 2 +- src/nix/run.cc | 10 +++++++--- src/nix/run.hh | 1 + 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 38482ed42..ae9be79a3 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -662,7 +662,7 @@ struct CmdDevelop : Common, MixEnvironment } } - runProgramInStore(store, shell, args, buildEnvironment.getSystem()); + runProgramInStore(store, true, shell, args, buildEnvironment.getSystem()); } }; diff --git a/src/nix/fmt.cc b/src/nix/fmt.cc index c85eacded..396c93dbb 100644 --- a/src/nix/fmt.cc +++ b/src/nix/fmt.cc @@ -49,7 +49,7 @@ struct CmdFmt : SourceExprCommand { } } - runProgramInStore(store, app.program, programArgs); + runProgramInStore(store, false, app.program, programArgs); }; }; diff --git a/src/nix/run.cc b/src/nix/run.cc index ea0a17897..d531f712d 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -25,6 +25,7 @@ std::string chrootHelperName = "__run_in_chroot"; namespace nix { void runProgramInStore(ref store, + bool search, const std::string & program, const Strings & args, std::optional system) @@ -58,7 +59,10 @@ void runProgramInStore(ref store, if (system) setPersonality(*system); - execvp(program.c_str(), stringsToCharPtrs(args).data()); + if (search) + execvp(program.c_str(), stringsToCharPtrs(args).data()); + else + execv(program.c_str(), stringsToCharPtrs(args).data()); throw SysError("unable to execute '%s'", program); } @@ -132,7 +136,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment Strings args; for (auto & arg : command) args.push_back(arg); - runProgramInStore(store, *command.begin(), args); + runProgramInStore(store, true, *command.begin(), args); } }; @@ -194,7 +198,7 @@ struct CmdRun : InstallableValueCommand Strings allArgs{app.program}; for (auto & i : args) allArgs.push_back(i); - runProgramInStore(store, app.program, allArgs); + runProgramInStore(store, false, app.program, allArgs); } }; diff --git a/src/nix/run.hh b/src/nix/run.hh index 97ddef19b..c62287e7e 100644 --- a/src/nix/run.hh +++ b/src/nix/run.hh @@ -6,6 +6,7 @@ namespace nix { void runProgramInStore(ref store, + bool search, const std::string & program, const Strings & args, std::optional system = std::nullopt); From f99e468640ab0eac7f07ac2b328222eb45dee8d8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 27 Nov 2023 08:48:11 -0500 Subject: [PATCH 014/141] Avoid `/` in documentation URLs They are redundant and look weird. --- doc/manual/_redirects | 18 ++++++++++++++---- doc/manual/src/SUMMARY.md.in | 14 +++++++------- .../{advanced-topics.md => index.md} | 0 doc/manual/src/architecture/architecture.md | 2 +- .../command-ref/{command-ref.md => index.md} | 0 doc/manual/src/contributing/hacking.md | 2 +- .../contributing/{contributing.md => index.md} | 0 .../installation/{installation.md => index.md} | 0 .../{package-management.md => index.md} | 0 .../src/protocols/{protocols.md => index.md} | 0 doc/manual/src/quick-start.md | 2 +- .../{release-notes.md => index.md} | 0 12 files changed, 24 insertions(+), 14 deletions(-) rename doc/manual/src/advanced-topics/{advanced-topics.md => index.md} (100%) rename doc/manual/src/command-ref/{command-ref.md => index.md} (100%) rename doc/manual/src/contributing/{contributing.md => index.md} (100%) rename doc/manual/src/installation/{installation.md => index.md} (100%) rename doc/manual/src/package-management/{package-management.md => index.md} (100%) rename doc/manual/src/protocols/{protocols.md => index.md} (100%) rename doc/manual/src/release-notes/{release-notes.md => index.md} (100%) diff --git a/doc/manual/_redirects b/doc/manual/_redirects index 4ea289d86..2038671d7 100644 --- a/doc/manual/_redirects +++ b/doc/manual/_redirects @@ -13,18 +13,28 @@ # conventions: # - always force (!) since this allows re-using file names # - group related paths to ease readability -# - always append new redirects to the end of the file +# - keep in alphabetical/wildcards-last order, which will reduce version control conflicts # - redirects that should have been there but are missing can be inserted where they belong +/advanced-topics/advanced-topics /advanced-topics 301! + +/command-ref/command-ref /command-ref 301! + +/contributing/contributing /contributing 301! + /expressions/expression-language /language/ 301! -/expressions/language-values /language/values 301! /expressions/language-constructs /language/constructs 301! /expressions/language-operators /language/operators 301! +/expressions/language-values /language/values 301! /expressions/* /language/:splat 301! +/installation/installation /installation 301! + /package-management/basic-package-mgmt /command-ref/nix-env 301! - /package-management/channels* /command-ref/nix-channel 301! - +/package-management/package-management /package-management 301! /package-management/s3-substituter* /command-ref/new-cli/nix3-help-stores#s3-binary-cache-store 301! +/protocols/protocols /protocols 301! + +/release-notes/release-notes /release-notes 301! diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 8e7b4eeab..686d3e8d7 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -2,7 +2,7 @@ - [Introduction](introduction.md) - [Quick Start](quick-start.md) -- [Installation](installation/installation.md) +- [Installation](installation/index.md) - [Supported Platforms](installation/supported-platforms.md) - [Installing a Binary Distribution](installation/installing-binary.md) - [Installing Nix from Source](installation/installing-source.md) @@ -31,11 +31,11 @@ - [Import From Derivation](language/import-from-derivation.md) - [Built-in Constants](language/builtin-constants.md) - [Built-in Functions](language/builtins.md) -- [Package Management](package-management/package-management.md) +- [Package Management](package-management/index.md) - [Profiles](package-management/profiles.md) - [Garbage Collection](package-management/garbage-collection.md) - [Garbage Collector Roots](package-management/garbage-collector-roots.md) -- [Advanced Topics](advanced-topics/advanced-topics.md) +- [Advanced Topics](advanced-topics/index.md) - [Sharing Packages Between Machines](package-management/sharing-packages.md) - [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md) - [Copying Closures via SSH](package-management/copy-closure.md) @@ -45,7 +45,7 @@ - [Tuning Cores and Jobs](advanced-topics/cores-vs-jobs.md) - [Verifying Build Reproducibility](advanced-topics/diff-hook.md) - [Using the `post-build-hook`](advanced-topics/post-build-hook.md) -- [Command Reference](command-ref/command-ref.md) +- [Command Reference](command-ref/index.md) - [Common Options](command-ref/opt-common.md) - [Common Environment Variables](command-ref/env-common.md) - [Main Commands](command-ref/main-commands.md) @@ -102,18 +102,18 @@ - [Channels](command-ref/files/channels.md) - [Default Nix expression](command-ref/files/default-nix-expression.md) - [Architecture and Design](architecture/architecture.md) -- [Protocols](protocols/protocols.md) +- [Protocols](protocols/index.md) - [Serving Tarball Flakes](protocols/tarball-fetcher.md) - [Derivation "ATerm" file format](protocols/derivation-aterm.md) - [Glossary](glossary.md) -- [Contributing](contributing/contributing.md) +- [Contributing](contributing/index.md) - [Hacking](contributing/hacking.md) - [Testing](contributing/testing.md) - [Documentation](contributing/documentation.md) - [Experimental Features](contributing/experimental-features.md) - [CLI guideline](contributing/cli-guideline.md) - [C++ style guide](contributing/cxx.md) -- [Release Notes](release-notes/release-notes.md) +- [Release Notes](release-notes/index.md) {{#include ./SUMMARY-rl-next.md}} - [Release 2.19 (2023-11-17)](release-notes/rl-2.19.md) - [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md) diff --git a/doc/manual/src/advanced-topics/advanced-topics.md b/doc/manual/src/advanced-topics/index.md similarity index 100% rename from doc/manual/src/advanced-topics/advanced-topics.md rename to doc/manual/src/advanced-topics/index.md diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 79429508f..2fec4ed20 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -52,7 +52,7 @@ The following [concept map] shows its main components (rectangles), the objects '---------------' ``` -At the top is the [command line interface](../command-ref/command-ref.md) that drives the underlying layers. +At the top is the [command line interface](../command-ref/index.md) that drives the underlying layers. The [Nix language](../language/index.md) evaluator transforms Nix expressions into self-contained *build plans*, which are used to derive *build results* from referenced *build inputs*. diff --git a/doc/manual/src/command-ref/command-ref.md b/doc/manual/src/command-ref/index.md similarity index 100% rename from doc/manual/src/command-ref/command-ref.md rename to doc/manual/src/command-ref/index.md diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 0a95334f7..9de5ad39b 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -10,7 +10,7 @@ $ cd nix The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. If you don't have it installed, follow the [installation instructions]. -[installation instructions]: ../installation/installation.md +[installation instructions]: ../installation/index.md ## Building Nix with flakes diff --git a/doc/manual/src/contributing/contributing.md b/doc/manual/src/contributing/index.md similarity index 100% rename from doc/manual/src/contributing/contributing.md rename to doc/manual/src/contributing/index.md diff --git a/doc/manual/src/installation/installation.md b/doc/manual/src/installation/index.md similarity index 100% rename from doc/manual/src/installation/installation.md rename to doc/manual/src/installation/index.md diff --git a/doc/manual/src/package-management/package-management.md b/doc/manual/src/package-management/index.md similarity index 100% rename from doc/manual/src/package-management/package-management.md rename to doc/manual/src/package-management/index.md diff --git a/doc/manual/src/protocols/protocols.md b/doc/manual/src/protocols/index.md similarity index 100% rename from doc/manual/src/protocols/protocols.md rename to doc/manual/src/protocols/index.md diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md index 1d2688ede..5f54abbde 100644 --- a/doc/manual/src/quick-start.md +++ b/doc/manual/src/quick-start.md @@ -13,7 +13,7 @@ to subsequent chapters. The install script will use `sudo`, so make sure you have sufficient rights. On Linux, `--daemon` can be omitted for a single-user install. - For other installation methods, see [here](installation/installation.md). + For other installation methods, see [here](installation/index.md). 1. See what installable packages are currently available in the channel: diff --git a/doc/manual/src/release-notes/release-notes.md b/doc/manual/src/release-notes/index.md similarity index 100% rename from doc/manual/src/release-notes/release-notes.md rename to doc/manual/src/release-notes/index.md From ea95327e72f5781295417b0eae46a5e351bebebd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 30 Nov 2023 16:16:17 +0100 Subject: [PATCH 015/141] Move restricted/pure-eval access control out of the evaluator and into the accessor --- src/libcmd/installables.cc | 7 +- src/libexpr/eval.cc | 103 +++++---------------- src/libexpr/eval.hh | 25 +++--- src/libexpr/parser.y | 19 +++- src/libexpr/primops.cc | 119 +++++++++++-------------- src/nix-build/nix-build.cc | 7 +- src/nix-instantiate/nix-instantiate.cc | 2 +- tests/functional/restricted.sh | 4 +- 8 files changed, 115 insertions(+), 171 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 6e670efea..6b3c82374 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -260,9 +260,10 @@ void SourceExprCommand::completeInstallable(AddCompletions & completions, std::s evalSettings.pureEval = false; auto state = getEvalState(); - Expr *e = state->parseExprFromFile( - resolveExprPath(state->checkSourcePath(lookupFileArg(*state, *file))) - ); + auto e = + state->parseExprFromFile( + resolveExprPath( + lookupFileArg(*state, *file))); Value root; state->eval(e, root); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7e68e6f9b..23ac349fe 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -509,7 +509,18 @@ EvalState::EvalState( , sOutputSpecified(symbols.create("outputSpecified")) , repair(NoRepair) , emptyBindings(0) - , rootFS(makeFSInputAccessor(CanonPath::root)) + , rootFS( + makeFSInputAccessor( + CanonPath::root, + evalSettings.restrictEval || evalSettings.pureEval + ? std::optional>(std::set()) + : std::nullopt, + [](const CanonPath & path) -> RestrictedPathError { + auto modeInformation = evalSettings.pureEval + ? "in pure evaluation mode (use '--impure' to override)" + : "in restricted mode"; + throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation); + })) , corepkgsFS(makeMemoryInputAccessor()) , internalFS(makeMemoryInputAccessor()) , derivationInternal{corepkgsFS->addFile( @@ -551,28 +562,10 @@ EvalState::EvalState( searchPath.elements.emplace_back(SearchPath::Elem::parse(i)); } - if (evalSettings.restrictEval || evalSettings.pureEval) { - allowedPaths = PathSet(); - - for (auto & i : searchPath.elements) { - auto r = resolveSearchPathPath(i.path); - if (!r) continue; - - auto path = std::move(*r); - - if (store->isInStore(path)) { - try { - StorePathSet closure; - store->computeFSClosure(store->toStorePath(path).first, closure); - for (auto & path : closure) - allowPath(path); - } catch (InvalidPath &) { - allowPath(path); - } - } else - allowPath(path); - } - } + /* Allow access to all paths in the search path. */ + if (rootFS->hasAccessControl()) + for (auto & i : searchPath.elements) + resolveSearchPathPath(i.path, true); corepkgsFS->addFile( CanonPath("fetchurl.nix"), @@ -590,14 +583,12 @@ EvalState::~EvalState() void EvalState::allowPath(const Path & path) { - if (allowedPaths) - allowedPaths->insert(path); + rootFS->allowPath(CanonPath(path)); } void EvalState::allowPath(const StorePath & storePath) { - if (allowedPaths) - allowedPaths->insert(store->toRealPath(storePath)); + rootFS->allowPath(CanonPath(store->toRealPath(storePath))); } void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & v) @@ -607,54 +598,6 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & mkStorePathString(storePath, v); } -SourcePath EvalState::checkSourcePath(const SourcePath & path_) -{ - // Don't check non-rootFS accessors, they're in a different namespace. - if (path_.accessor != ref(rootFS)) return path_; - - if (!allowedPaths) return path_; - - auto i = resolvedPaths.find(path_.path.abs()); - if (i != resolvedPaths.end()) - return i->second; - - bool found = false; - - /* First canonicalize the path without symlinks, so we make sure an - * attacker can't append ../../... to a path that would be in allowedPaths - * and thus leak symlink targets. - */ - Path abspath = canonPath(path_.path.abs()); - - for (auto & i : *allowedPaths) { - if (isDirOrInDir(abspath, i)) { - found = true; - break; - } - } - - if (!found) { - auto modeInformation = evalSettings.pureEval - ? "in pure eval mode (use '--impure' to override)" - : "in restricted mode"; - throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", abspath, modeInformation); - } - - /* Resolve symlinks. */ - debug("checking access to '%s'", abspath); - SourcePath path = rootPath(CanonPath(canonPath(abspath, true))); - - for (auto & i : *allowedPaths) { - if (isDirOrInDir(path.path.abs(), i)) { - resolvedPaths.insert_or_assign(path_.path.abs(), path); - return path; - } - } - - throw RestrictedPathError("access to canonical path '%1%' is forbidden in restricted mode", path); -} - - void EvalState::checkURI(const std::string & uri) { if (!evalSettings.restrictEval) return; @@ -674,12 +617,12 @@ void EvalState::checkURI(const std::string & uri) /* If the URI is a path, then check it against allowedPaths as well. */ if (hasPrefix(uri, "/")) { - checkSourcePath(rootPath(CanonPath(uri))); + rootFS->checkAllowed(CanonPath(uri)); return; } if (hasPrefix(uri, "file://")) { - checkSourcePath(rootPath(CanonPath(std::string(uri, 7)))); + rootFS->checkAllowed(CanonPath(uri.substr(7))); return; } @@ -1181,10 +1124,8 @@ Value * ExprPath::maybeThunk(EvalState & state, Env & env) } -void EvalState::evalFile(const SourcePath & path_, Value & v, bool mustBeTrivial) +void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial) { - auto path = checkSourcePath(path_); - FileEvalCache::iterator i; if ((i = fileEvalCache.find(path)) != fileEvalCache.end()) { v = i->second; @@ -1205,7 +1146,7 @@ void EvalState::evalFile(const SourcePath & path_, Value & v, bool mustBeTrivial e = j->second; if (!e) - e = parseExprFromFile(checkSourcePath(resolvedPath)); + e = parseExprFromFile(resolvedPath); fileParseCache[resolvedPath] = e; diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 9a92992c1..ee7bdda0d 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -217,12 +217,6 @@ public: */ RepairFlag repair; - /** - * The allowed filesystem paths in restricted or pure evaluation - * mode. - */ - std::optional allowedPaths; - Bindings emptyBindings; /** @@ -396,12 +390,6 @@ public: */ void allowAndSetStorePathString(const StorePath & storePath, Value & v); - /** - * Check whether access to a path is allowed and throw an error if - * not. Otherwise return the canonicalised path. - */ - SourcePath checkSourcePath(const SourcePath & path); - void checkURI(const std::string & uri); /** @@ -445,13 +433,15 @@ public: SourcePath findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos); /** - * Try to resolve a search path value (not the optional key part) + * Try to resolve a search path value (not the optional key part). * * If the specified search path element is a URI, download it. * * If it is not found, return `std::nullopt` */ - std::optional resolveSearchPathPath(const SearchPath::Path & path); + std::optional resolveSearchPathPath( + const SearchPath::Path & elem, + bool initAccessControl = false); /** * Evaluate an expression to normal form @@ -756,6 +746,13 @@ public: */ [[nodiscard]] StringMap realiseContext(const NixStringContext & context); + /* Call the binary path filter predicate used builtins.path etc. */ + bool callPathFilter( + Value * filterFun, + const SourcePath & path, + std::string_view pathArg, + PosIdx pos); + private: /** diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index f6cf1f689..58fc580fc 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -783,7 +783,7 @@ SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_ } -std::optional EvalState::resolveSearchPathPath(const SearchPath::Path & value0) +std::optional EvalState::resolveSearchPathPath(const SearchPath::Path & value0, bool initAccessControl) { auto & value = value0.s; auto i = searchPathResolved.find(value); @@ -800,7 +800,6 @@ std::optional EvalState::resolveSearchPathPath(const SearchPath::Pa logWarning({ .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value) }); - res = std::nullopt; } } @@ -814,6 +813,20 @@ std::optional EvalState::resolveSearchPathPath(const SearchPath::Pa else { auto path = absPath(value); + + /* Allow access to paths in the search path. */ + if (initAccessControl) { + allowPath(path); + if (store->isInStore(path)) { + try { + StorePathSet closure; + store->computeFSClosure(store->toStorePath(path).first, closure); + for (auto & p : closure) + allowPath(p); + } catch (InvalidPath &) { } + } + } + if (pathExists(path)) res = { path }; else { @@ -829,7 +842,7 @@ std::optional EvalState::resolveSearchPathPath(const SearchPath::Pa else debug("failed to resolve search path element '%s'", value); - searchPathResolved[value] = res; + searchPathResolved.emplace(value, res); return res; } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index ebf2549e4..0f7706563 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -15,6 +15,7 @@ #include "value-to-json.hh" #include "value-to-xml.hh" #include "primops.hh" +#include "fs-input-accessor.hh" #include #include @@ -90,8 +91,8 @@ StringMap EvalState::realiseContext(const NixStringContext & context) for (auto & [outputName, outputPath] : outputs) { /* Add the output of this derivations to the allowed paths. */ - if (allowedPaths) { - allowPath(outputPath); + if (rootFS->hasAccessControl()) { + allowPath(store->toRealPath(outputPath)); } /* Get all the output paths corresponding to the placeholders we had */ if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { @@ -110,27 +111,19 @@ StringMap EvalState::realiseContext(const NixStringContext & context) return res; } -struct RealisePathFlags { - // Whether to check that the path is allowed in pure eval mode - bool checkForPureEval = true; -}; - -static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, const RealisePathFlags flags = {}) +static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v) { NixStringContext context; auto path = state.coerceToPath(noPos, v, context, "while realising the context of a path"); try { - if (!context.empty()) { + if (!context.empty() && path.accessor == state.rootFS) { auto rewrites = state.realiseContext(context); auto realPath = state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context); return {path.accessor, CanonPath(realPath)}; - } - - return flags.checkForPureEval - ? state.checkSourcePath(path) - : path; + } else + return path; } catch (Error & e) { e.addTrace(state.positions[pos], "while realising the context of path '%s'", path); throw; @@ -1493,7 +1486,7 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args, })); NixStringContext context; - auto path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'")).path; + auto path = state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'").path; /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink directly in the store. The latter condition is necessary so e.g. nix-push does the right thing. */ @@ -1535,12 +1528,7 @@ static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args, { auto & arg = *args[0]; - /* We don’t check the path right now, because we don’t want to - throw if the path isn’t allowed, but just return false (and we - can’t just catch the exception here because we still want to - throw if something in the evaluation of `arg` tries to - access an unauthorized path). */ - auto path = realisePath(state, pos, arg, { .checkForPureEval = false }); + auto path = realisePath(state, pos, arg); /* SourcePath doesn't know about trailing slash. */ auto mustBeDir = arg.type() == nString @@ -1548,14 +1536,9 @@ static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args, || arg.string_view().ends_with("/.")); try { - auto checked = state.checkSourcePath(path); - auto st = checked.maybeLstat(); + auto st = path.maybeLstat(); auto exists = st && (!mustBeDir || st->type == SourceAccessor::tDirectory); v.mkBool(exists); - } catch (SysError & e) { - /* Don't give away info from errors while canonicalising - ‘path’ in restricted mode. */ - v.mkBool(false); } catch (RestrictedPathError & e) { v.mkBool(false); } @@ -1699,7 +1682,7 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V auto path = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.findFile"); - v.mkPath(state.checkSourcePath(state.findFile(searchPath, path, pos))); + v.mkPath(state.findFile(searchPath, path, pos)); } static RegisterPrimOp primop_findFile(PrimOp { @@ -2178,11 +2161,35 @@ static RegisterPrimOp primop_toFile({ .fun = prim_toFile, }); +bool EvalState::callPathFilter( + Value * filterFun, + const SourcePath & path, + std::string_view pathArg, + PosIdx pos) +{ + auto st = path.lstat(); + + /* Call the filter function. The first argument is the path, the + second is a string indicating the type of the file. */ + Value arg1; + arg1.mkString(pathArg); + + Value arg2; + // assert that type is not "unknown" + arg2.mkString(fileTypeToString(st.type)); + + Value * args []{&arg1, &arg2}; + Value res; + callFunction(*filterFun, 2, args, res, pos); + + return forceBool(res, pos, "while evaluating the return value of the path filter function"); +} + static void addPath( EvalState & state, const PosIdx pos, std::string_view name, - Path path, + SourcePath path, Value * filterFun, FileIngestionMethod method, const std::optional expectedHash, @@ -2190,48 +2197,29 @@ static void addPath( const NixStringContext & context) { try { - // FIXME: handle CA derivation outputs (where path needs to - // be rewritten to the actual output). - auto rewrites = state.realiseContext(context); - path = state.toRealPath(rewriteStrings(path, rewrites), context); - StorePathSet refs; - if (state.store->isInStore(path)) { + if (path.accessor == state.rootFS && state.store->isInStore(path.path.abs())) { + // FIXME: handle CA derivation outputs (where path needs to + // be rewritten to the actual output). + auto rewrites = state.realiseContext(context); + path = {state.rootFS, CanonPath(state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context))}; + try { - auto [storePath, subPath] = state.store->toStorePath(path); + auto [storePath, subPath] = state.store->toStorePath(path.path.abs()); // FIXME: we should scanForReferences on the path before adding it refs = state.store->queryPathInfo(storePath)->references; - path = state.store->toRealPath(storePath) + subPath; + path = {state.rootFS, CanonPath(state.store->toRealPath(storePath) + subPath)}; } catch (Error &) { // FIXME: should be InvalidPathError } } - path = evalSettings.pureEval && expectedHash - ? path - : state.checkSourcePath(state.rootPath(CanonPath(path))).path.abs(); - - PathFilter filter = filterFun ? ([&](const Path & path) { - auto st = lstat(path); - - /* Call the filter function. The first argument is the path, - the second is a string indicating the type of the file. */ - Value arg1; - arg1.mkString(path); - - Value arg2; - arg2.mkString( - S_ISREG(st.st_mode) ? "regular" : - S_ISDIR(st.st_mode) ? "directory" : - S_ISLNK(st.st_mode) ? "symlink" : - "unknown" /* not supported, will fail! */); - - Value * args []{&arg1, &arg2}; - Value res; - state.callFunction(*filterFun, 2, args, res, pos); - - return state.forceBool(res, pos, "while evaluating the return value of the path filter function"); - }) : defaultPathFilter; + std::unique_ptr filter; + if (filterFun) + filter = std::make_unique([&](const Path & p) { + auto p2 = CanonPath(p); + return state.callPathFilter(filterFun, {path.accessor, p2}, p2.abs(), pos); + }); std::optional expectedStorePath; if (expectedHash) @@ -2242,7 +2230,7 @@ static void addPath( }); if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { - auto dstPath = state.rootPath(CanonPath(path)).fetchToStore(state.store, name, method, &filter, state.repair); + auto dstPath = path.fetchToStore(state.store, name, method, filter.get(), state.repair); if (expectedHash && expectedStorePath != dstPath) state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path)); state.allowAndSetStorePathString(dstPath, v); @@ -2261,7 +2249,8 @@ static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * arg auto path = state.coerceToPath(pos, *args[1], context, "while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'"); state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource"); - addPath(state, pos, path.baseName(), path.path.abs(), args[0], FileIngestionMethod::Recursive, std::nullopt, v, context); + + addPath(state, pos, path.baseName(), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v, context); } static RegisterPrimOp primop_filterSource({ @@ -2356,7 +2345,7 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value if (name.empty()) name = path->baseName(); - addPath(state, pos, name, path->path.abs(), filterFun, method, expectedHash, v, context); + addPath(state, pos, name, *path, filterFun, method, expectedHash, v, context); } static RegisterPrimOp primop_path({ diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 75ce12a8c..e2986bfe0 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -310,8 +310,11 @@ static void main_nix_build(int argc, char * * argv) else /* If we're in a #! script, interpret filenames relative to the script. */ - exprs.push_back(state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state, - inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i))))); + exprs.push_back( + state->parseExprFromFile( + resolveExprPath( + lookupFileArg(*state, + inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i)))); } } diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index c67409e89..86b9be17d 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -183,7 +183,7 @@ static int main_nix_instantiate(int argc, char * * argv) for (auto & i : files) { Expr * e = fromArgs ? state->parseExprFromString(i, state->rootPath(CanonPath::fromCwd())) - : state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state, i)))); + : state->parseExprFromFile(resolveExprPath(lookupFileArg(*state, i))); processExpr(*state, attrPaths, parseOnly, strict, autoArgs, evalOnly, outputKind, xmlOutputSourceLocation, e); } diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh index 197ae7a10..b8deceacc 100644 --- a/tests/functional/restricted.sh +++ b/tests/functional/restricted.sh @@ -14,8 +14,8 @@ nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I sr (! nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel') nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel' -I src=../../src -(! nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in ') -nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in ' -I src=. +(! nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile ') +nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile ' -I src=. p=$(nix eval --raw --expr "builtins.fetchurl file://$(pwd)/restricted.sh" --impure --restrict-eval --allowed-uris "file://$(pwd)") cmp $p restricted.sh From 305939655a6cd680997981ca6077d4ce7f957984 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 30 Nov 2023 16:28:33 +0100 Subject: [PATCH 016/141] Remove superfluous use of hasAccessControl() --- src/libexpr/primops.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 0f7706563..c442de986 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -91,9 +91,8 @@ StringMap EvalState::realiseContext(const NixStringContext & context) for (auto & [outputName, outputPath] : outputs) { /* Add the output of this derivations to the allowed paths. */ - if (rootFS->hasAccessControl()) { - allowPath(store->toRealPath(outputPath)); - } + allowPath(store->toRealPath(outputPath)); + /* Get all the output paths corresponding to the placeholders we had */ if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { res.insert_or_assign( From 43d9fb6cf180c421be17b4247f5dd032cf4843f5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 30 Nov 2023 16:44:54 +0100 Subject: [PATCH 017/141] Remove InputAccessor::root() --- src/libexpr/value.hh | 7 +++---- src/libfetchers/fetchers.cc | 2 +- src/libfetchers/input-accessor.cc | 7 +------ src/libfetchers/input-accessor.hh | 7 +++++-- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index bcff8ae55..72a3a2b32 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -424,10 +424,9 @@ public: SourcePath path() const { assert(internalType == tPath); - return SourcePath { - .accessor = ref(_path.accessor->shared_from_this()), - .path = CanonPath(CanonPath::unchecked_t(), _path.path) - }; + return SourcePath( + ref(_path.accessor->shared_from_this()), + CanonPath(CanonPath::unchecked_t(), _path.path)); } std::string_view string_view() const diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 60208619e..5fd9e069f 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -374,7 +374,7 @@ void InputScheme::clone(const Input & input, const Path & destDir) const std::pair InputScheme::fetch(ref store, const Input & input) { auto [accessor, input2] = getAccessor(store, input); - auto storePath = accessor->root().fetchToStore(store, input2.getName()); + auto storePath = SourcePath(accessor).fetchToStore(store, input2.getName()); return {storePath, input2}; } diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc index 85dc4609f..f54a5a6fd 100644 --- a/src/libfetchers/input-accessor.cc +++ b/src/libfetchers/input-accessor.cc @@ -53,11 +53,6 @@ StorePath InputAccessor::fetchToStore( return storePath; } -SourcePath InputAccessor::root() -{ - return {ref(shared_from_this()), CanonPath::root}; -} - std::ostream & operator << (std::ostream & str, const SourcePath & path) { str << path.to_string(); @@ -88,7 +83,7 @@ SourcePath SourcePath::parent() const SourcePath SourcePath::resolveSymlinks() const { - auto res = accessor->root(); + auto res = SourcePath(accessor); int linksAllowed = 1024; diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh index 26d17f064..d5ac238b1 100644 --- a/src/libfetchers/input-accessor.hh +++ b/src/libfetchers/input-accessor.hh @@ -36,8 +36,6 @@ struct InputAccessor : virtual SourceAccessor, std::enable_shared_from_this accessor; CanonPath path; + SourcePath(ref accessor, CanonPath path = CanonPath::root) + : accessor(std::move(accessor)) + , path(std::move(path)) + { } + std::string_view baseName() const; /** From 44d21f6ef9783bed8812d39ff7b1a28a4883f84b Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 30 Nov 2023 20:36:50 +0100 Subject: [PATCH 018/141] keep generated documentation in a separate directory - helps navigating the code as it highlights which files are generated - makes it less error prone when working incrementally (although this should be just fixed by building out of tree) --- .gitignore | 2 +- .../manual/src/store/types/index.md.in | 4 ---- src/nix/local.mk | 19 ++++++++++++++++--- src/nix/main.cc | 2 +- src/nix/profile.md | 2 +- 5 files changed, 19 insertions(+), 10 deletions(-) rename src/nix/help-stores.md => doc/manual/src/store/types/index.md.in (99%) diff --git a/.gitignore b/.gitignore index 13a4dfb75..bcf9c4a01 100644 --- a/.gitignore +++ b/.gitignore @@ -54,7 +54,7 @@ perl/Makefile.config /src/nix/nix -/src/nix/doc +/src/nix/generated-doc # /src/nix-env/ /src/nix-env/nix-env diff --git a/src/nix/help-stores.md b/doc/manual/src/store/types/index.md.in similarity index 99% rename from src/nix/help-stores.md rename to doc/manual/src/store/types/index.md.in index 47ba9b94d..bb166a1fc 100644 --- a/src/nix/help-stores.md +++ b/doc/manual/src/store/types/index.md.in @@ -1,5 +1,3 @@ -R"( - Nix supports different types of stores. These are described below. ## Store URL format @@ -42,5 +40,3 @@ store as follows: * Otherwise, use the [local store](#local-store) `/nix/store`. @stores@ - -)" diff --git a/src/nix/local.mk b/src/nix/local.mk index 57f8259c4..a21aa705f 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -31,10 +31,23 @@ src/nix/develop.cc: src/nix/get-env.sh.gen.hh src/nix-channel/nix-channel.cc: src/nix-channel/unpack-channel.nix.gen.hh -src/nix/main.cc: doc/manual/generate-manpage.nix.gen.hh doc/manual/utils.nix.gen.hh doc/manual/generate-settings.nix.gen.hh doc/manual/generate-store-info.nix.gen.hh +src/nix/main.cc: \ + doc/manual/generate-manpage.nix.gen.hh \ + doc/manual/utils.nix.gen.hh doc/manual/generate-settings.nix.gen.hh \ + doc/manual/generate-store-info.nix.gen.hh \ + src/nix/generated-doc/help-stores.md -src/nix/doc/files/%.md: doc/manual/src/command-ref/files/%.md +src/nix/generated-doc/files/%.md: doc/manual/src/command-ref/files/%.md @mkdir -p $$(dirname $@) @cp $< $@ -src/nix/profile.cc: src/nix/profile.md src/nix/doc/files/profiles.md.gen.hh +src/nix/profile.cc: src/nix/profile.md src/nix/generated-doc/files/profiles.md.gen.hh + +src/nix/generated-doc/help-stores.md: doc/manual/src/store/types/index.md.in + @mkdir -p $$(dirname $@) + @echo 'R"(' >> $@.tmp + @echo >> $@.tmp + @cat $^ >> $@.tmp + @echo >> $@.tmp + @echo ')"' >> $@.tmp + @mv $@.tmp $@ diff --git a/src/nix/main.cc b/src/nix/main.cc index 2a6c2f478..49e637fb0 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -297,7 +297,7 @@ struct CmdHelpStores : Command std::string doc() override { return - #include "help-stores.md" + #include "generated-doc/help-stores.md" ; } diff --git a/src/nix/profile.md b/src/nix/profile.md index bd13f906f..9b2f86f4a 100644 --- a/src/nix/profile.md +++ b/src/nix/profile.md @@ -11,7 +11,7 @@ them to be rolled back easily. )"" -#include "doc/files/profiles.md.gen.hh" +#include "generated-doc/files/profiles.md.gen.hh" R""( From d5ffc94f336fc4032dd4009c14c148e390e10e16 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 30 Nov 2023 21:41:47 +0100 Subject: [PATCH 019/141] use lookup paths in helper expressions consistently this makes the files in question a bit more independent of source location. to find where the value is set and how it's wired up: rg nix=doc/manual --- doc/manual/generate-builtin-constants.nix | 2 +- doc/manual/generate-builtins.nix | 2 +- doc/manual/generate-manpage.nix | 30 +++++++++++++++---- doc/manual/generate-settings.nix | 2 +- doc/manual/generate-store-info.nix | 4 +-- doc/manual/generate-xp-features-shortlist.nix | 2 +- doc/manual/generate-xp-features.nix | 2 +- 7 files changed, 32 insertions(+), 12 deletions(-) diff --git a/doc/manual/generate-builtin-constants.nix b/doc/manual/generate-builtin-constants.nix index 8af80a02c..cccd1e279 100644 --- a/doc/manual/generate-builtin-constants.nix +++ b/doc/manual/generate-builtin-constants.nix @@ -1,6 +1,6 @@ let inherit (builtins) concatStringsSep attrValues mapAttrs; - inherit (import ./utils.nix) optionalString squash; + inherit (import ) optionalString squash; in builtinsInfo: diff --git a/doc/manual/generate-builtins.nix b/doc/manual/generate-builtins.nix index 813a287f5..05cae1c46 100644 --- a/doc/manual/generate-builtins.nix +++ b/doc/manual/generate-builtins.nix @@ -1,6 +1,6 @@ let inherit (builtins) concatStringsSep attrValues mapAttrs; - inherit (import ./utils.nix) optionalString squash; + inherit (import ) optionalString squash; in builtinsInfo: diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 14136016d..c4b9d1335 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -1,9 +1,29 @@ let inherit (builtins) - attrNames attrValues fromJSON listToAttrs mapAttrs groupBy - concatStringsSep concatMap length lessThan replaceStrings sort; - inherit (import ) attrsToList concatStrings optionalString filterAttrs trim squash unique; - showStoreDocs = import ./generate-store-info.nix; + attrNames + attrValues + concatMap + concatStringsSep + fromJSON + groupBy + length + lessThan + listToAttrs + mapAttrs + match + replaceStrings + sort + ; + inherit (import ) + attrsToList + concatStrings + filterAttrs + optionalString + squash + trim + unique + ; + showStoreDocs = import ; in inlineHTML: commandDump: @@ -97,7 +117,7 @@ let ${optionalString (cat != "") "## ${cat}"} ${concatStringsSep "\n" (attrValues (mapAttrs showOption opts))} - ''; + ''; showOption = name: option: let result = trim '' diff --git a/doc/manual/generate-settings.nix b/doc/manual/generate-settings.nix index 8736bb793..3add10075 100644 --- a/doc/manual/generate-settings.nix +++ b/doc/manual/generate-settings.nix @@ -1,6 +1,6 @@ let inherit (builtins) attrValues concatStringsSep isAttrs isBool mapAttrs; - inherit (import ./utils.nix) concatStrings indent optionalString squash; + inherit (import ) concatStrings indent optionalString squash; in # `inlineHTML` is a hack to accommodate inconsistent output from `lowdown` diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix index 36215aadf..73defcb71 100644 --- a/doc/manual/generate-store-info.nix +++ b/doc/manual/generate-store-info.nix @@ -1,7 +1,7 @@ let inherit (builtins) attrValues mapAttrs; - inherit (import ./utils.nix) concatStrings optionalString; - showSettings = import ./generate-settings.nix; + inherit (import ) concatStrings optionalString; + showSettings = import ; in inlineHTML: storesInfo: diff --git a/doc/manual/generate-xp-features-shortlist.nix b/doc/manual/generate-xp-features-shortlist.nix index 30e211c96..ec09f4b75 100644 --- a/doc/manual/generate-xp-features-shortlist.nix +++ b/doc/manual/generate-xp-features-shortlist.nix @@ -1,5 +1,5 @@ with builtins; -with import ./utils.nix; +with import ; let showExperimentalFeature = name: doc: diff --git a/doc/manual/generate-xp-features.nix b/doc/manual/generate-xp-features.nix index adb94355c..fc7c7d4cf 100644 --- a/doc/manual/generate-xp-features.nix +++ b/doc/manual/generate-xp-features.nix @@ -1,5 +1,5 @@ with builtins; -with import ./utils.nix; +with import ; let showExperimentalFeature = name: doc: From 8cafc754d845529a78595d1196769257ee23ca56 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 30 Nov 2023 21:54:53 +0100 Subject: [PATCH 020/141] Move access control from FSInputAccessor to FilteringInputAccessor --- src/libexpr/eval.cc | 23 +++--- src/libexpr/eval.hh | 3 +- src/libfetchers/filtering-input-accessor.cc | 83 +++++++++++++++++++++ src/libfetchers/filtering-input-accessor.hh | 73 ++++++++++++++++++ src/libfetchers/fs-input-accessor.cc | 77 +++---------------- src/libfetchers/fs-input-accessor.hh | 22 +----- src/libfetchers/git.cc | 6 +- 7 files changed, 191 insertions(+), 96 deletions(-) create mode 100644 src/libfetchers/filtering-input-accessor.cc create mode 100644 src/libfetchers/filtering-input-accessor.hh diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 23ac349fe..841c223cd 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -14,6 +14,7 @@ #include "profiles.hh" #include "print.hh" #include "fs-input-accessor.hh" +#include "filtering-input-accessor.hh" #include "memory-input-accessor.hh" #include "signals.hh" #include "gc-small-vector.hh" @@ -510,17 +511,15 @@ EvalState::EvalState( , repair(NoRepair) , emptyBindings(0) , rootFS( - makeFSInputAccessor( - CanonPath::root, - evalSettings.restrictEval || evalSettings.pureEval - ? std::optional>(std::set()) - : std::nullopt, + evalSettings.restrictEval || evalSettings.pureEval + ? ref(AllowListInputAccessor::create(makeFSInputAccessor(CanonPath::root), {}, [](const CanonPath & path) -> RestrictedPathError { auto modeInformation = evalSettings.pureEval ? "in pure evaluation mode (use '--impure' to override)" : "in restricted mode"; throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation); })) + : makeFSInputAccessor(CanonPath::root)) , corepkgsFS(makeMemoryInputAccessor()) , internalFS(makeMemoryInputAccessor()) , derivationInternal{corepkgsFS->addFile( @@ -563,7 +562,7 @@ EvalState::EvalState( } /* Allow access to all paths in the search path. */ - if (rootFS->hasAccessControl()) + if (rootFS.dynamic_pointer_cast()) for (auto & i : searchPath.elements) resolveSearchPathPath(i.path, true); @@ -583,12 +582,14 @@ EvalState::~EvalState() void EvalState::allowPath(const Path & path) { - rootFS->allowPath(CanonPath(path)); + if (auto rootFS2 = rootFS.dynamic_pointer_cast()) + rootFS2->allowPath(CanonPath(path)); } void EvalState::allowPath(const StorePath & storePath) { - rootFS->allowPath(CanonPath(store->toRealPath(storePath))); + if (auto rootFS2 = rootFS.dynamic_pointer_cast()) + rootFS2->allowPath(CanonPath(store->toRealPath(storePath))); } void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & v) @@ -617,12 +618,14 @@ void EvalState::checkURI(const std::string & uri) /* If the URI is a path, then check it against allowedPaths as well. */ if (hasPrefix(uri, "/")) { - rootFS->checkAllowed(CanonPath(uri)); + if (auto rootFS2 = rootFS.dynamic_pointer_cast()) + rootFS2->checkAccess(CanonPath(uri)); return; } if (hasPrefix(uri, "file://")) { - rootFS->checkAllowed(CanonPath(uri.substr(7))); + if (auto rootFS2 = rootFS.dynamic_pointer_cast()) + rootFS2->checkAccess(CanonPath(uri.substr(7))); return; } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index ee7bdda0d..f3f6d35b9 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -30,7 +30,6 @@ class EvalState; class StorePath; struct SingleDerivedPath; enum RepairFlag : bool; -struct FSInputAccessor; struct MemoryInputAccessor; @@ -222,7 +221,7 @@ public: /** * The accessor for the root filesystem. */ - const ref rootFS; + const ref rootFS; /** * The in-memory filesystem for paths. diff --git a/src/libfetchers/filtering-input-accessor.cc b/src/libfetchers/filtering-input-accessor.cc new file mode 100644 index 000000000..5ae416fd3 --- /dev/null +++ b/src/libfetchers/filtering-input-accessor.cc @@ -0,0 +1,83 @@ +#include "filtering-input-accessor.hh" + +namespace nix { + +std::string FilteringInputAccessor::readFile(const CanonPath & path) +{ + checkAccess(path); + return next->readFile(prefix + path); +} + +bool FilteringInputAccessor::pathExists(const CanonPath & path) +{ + return isAllowed(path) && next->pathExists(prefix + path); +} + +std::optional FilteringInputAccessor::maybeLstat(const CanonPath & path) +{ + checkAccess(path); + return next->maybeLstat(prefix + path); +} + +InputAccessor::DirEntries FilteringInputAccessor::readDirectory(const CanonPath & path) +{ + checkAccess(path); + DirEntries entries; + for (auto & entry : next->readDirectory(prefix + path)) { + if (isAllowed(path + entry.first)) + entries.insert(std::move(entry)); + } + return entries; +} + +std::string FilteringInputAccessor::readLink(const CanonPath & path) +{ + checkAccess(path); + return next->readLink(prefix + path); +} + +std::string FilteringInputAccessor::showPath(const CanonPath & path) +{ + return next->showPath(prefix + path); +} + +void FilteringInputAccessor::checkAccess(const CanonPath & path) +{ + if (!isAllowed(path)) + throw makeNotAllowedError + ? makeNotAllowedError(path) + : RestrictedPathError("access to path '%s' is forbidden", showPath(path)); +} + +struct AllowListInputAccessorImpl : AllowListInputAccessor +{ + std::set allowedPaths; + + AllowListInputAccessorImpl( + ref next, + std::set && allowedPaths, + MakeNotAllowedError && makeNotAllowedError) + : AllowListInputAccessor(SourcePath(next), std::move(makeNotAllowedError)) + , allowedPaths(std::move(allowedPaths)) + { } + + bool isAllowed(const CanonPath & path) override + { + return path.isAllowed(allowedPaths); + } + + void allowPath(CanonPath path) override + { + allowedPaths.insert(std::move(path)); + } +}; + +ref AllowListInputAccessor::create( + ref next, + std::set && allowedPaths, + MakeNotAllowedError && makeNotAllowedError) +{ + return make_ref(next, std::move(allowedPaths), std::move(makeNotAllowedError)); +} + +} diff --git a/src/libfetchers/filtering-input-accessor.hh b/src/libfetchers/filtering-input-accessor.hh new file mode 100644 index 000000000..209d26974 --- /dev/null +++ b/src/libfetchers/filtering-input-accessor.hh @@ -0,0 +1,73 @@ +#pragma once + +#include "input-accessor.hh" + +namespace nix { + +/** + * A function that should throw an exception of type + * `RestrictedPathError` explaining that access to `path` is + * forbidden. + */ +typedef std::function MakeNotAllowedError; + +/** + * An abstract wrapping `InputAccessor` that performs access + * control. Subclasses should override `checkAccess()` to implement an + * access control policy. + */ +struct FilteringInputAccessor : InputAccessor +{ + ref next; + CanonPath prefix; + MakeNotAllowedError makeNotAllowedError; + + FilteringInputAccessor(const SourcePath & src, MakeNotAllowedError && makeNotAllowedError) + : next(src.accessor) + , prefix(src.path) + , makeNotAllowedError(std::move(makeNotAllowedError)) + { } + + std::string readFile(const CanonPath & path) override; + + bool pathExists(const CanonPath & path) override; + + std::optional maybeLstat(const CanonPath & path) override; + + DirEntries readDirectory(const CanonPath & path) override; + + std::string readLink(const CanonPath & path) override; + + std::string showPath(const CanonPath & path) override; + + /** + * Call `makeNotAllowedError` to throw a `RestrictedPathError` + * exception if `isAllowed()` returns `false` for `path`. + */ + void checkAccess(const CanonPath & path); + + /** + * Return `true` iff access to path is allowed. + */ + virtual bool isAllowed(const CanonPath & path) = 0; +}; + +/** + * A wrapping `InputAccessor` that checks paths against an allow-list. + */ +struct AllowListInputAccessor : public FilteringInputAccessor +{ + /** + * Grant access to the specified path. + */ + virtual void allowPath(CanonPath path) = 0; + + static ref create( + ref next, + std::set && allowedPaths, + MakeNotAllowedError && makeNotAllowedError); + + using FilteringInputAccessor::FilteringInputAccessor; +}; + +} diff --git a/src/libfetchers/fs-input-accessor.cc b/src/libfetchers/fs-input-accessor.cc index 2efee932d..c3d8d273c 100644 --- a/src/libfetchers/fs-input-accessor.cc +++ b/src/libfetchers/fs-input-accessor.cc @@ -4,19 +4,12 @@ namespace nix { -struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor +struct FSInputAccessor : InputAccessor, PosixSourceAccessor { CanonPath root; - std::optional> allowedPaths; - MakeNotAllowedError makeNotAllowedError; - FSInputAccessorImpl( - const CanonPath & root, - std::optional> && allowedPaths, - MakeNotAllowedError && makeNotAllowedError) + FSInputAccessor(const CanonPath & root) : root(root) - , allowedPaths(std::move(allowedPaths)) - , makeNotAllowedError(std::move(makeNotAllowedError)) { displayPrefix = root.isRoot() ? "" : root.abs(); } @@ -27,39 +20,30 @@ struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor std::function sizeCallback) override { auto absPath = makeAbsPath(path); - checkAllowed(absPath); PosixSourceAccessor::readFile(absPath, sink, sizeCallback); } bool pathExists(const CanonPath & path) override { - auto absPath = makeAbsPath(path); - return isAllowed(absPath) && PosixSourceAccessor::pathExists(absPath); + return PosixSourceAccessor::pathExists(makeAbsPath(path)); } std::optional maybeLstat(const CanonPath & path) override { - auto absPath = makeAbsPath(path); - checkAllowed(absPath); - return PosixSourceAccessor::maybeLstat(absPath); + return PosixSourceAccessor::maybeLstat(makeAbsPath(path)); } DirEntries readDirectory(const CanonPath & path) override { - auto absPath = makeAbsPath(path); - checkAllowed(absPath); DirEntries res; - for (auto & entry : PosixSourceAccessor::readDirectory(absPath)) - if (isAllowed(absPath + entry.first)) - res.emplace(entry); + for (auto & entry : PosixSourceAccessor::readDirectory(makeAbsPath(path))) + res.emplace(entry); return res; } std::string readLink(const CanonPath & path) override { - auto absPath = makeAbsPath(path); - checkAllowed(absPath); - return PosixSourceAccessor::readLink(absPath); + return PosixSourceAccessor::readLink(makeAbsPath(path)); } CanonPath makeAbsPath(const CanonPath & path) @@ -67,59 +51,22 @@ struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor return root + path; } - void checkAllowed(const CanonPath & absPath) override - { - if (!isAllowed(absPath)) - throw makeNotAllowedError - ? makeNotAllowedError(absPath) - : RestrictedPathError("access to path '%s' is forbidden", absPath); - } - - bool isAllowed(const CanonPath & absPath) - { - if (!absPath.isWithin(root)) - return false; - - if (allowedPaths) { - auto p = absPath.removePrefix(root); - if (!p.isAllowed(*allowedPaths)) - return false; - } - - return true; - } - - void allowPath(CanonPath path) override - { - if (allowedPaths) - allowedPaths->insert(std::move(path)); - } - - bool hasAccessControl() override - { - return (bool) allowedPaths; - } - std::optional getPhysicalPath(const CanonPath & path) override { return makeAbsPath(path); } }; -ref makeFSInputAccessor( - const CanonPath & root, - std::optional> && allowedPaths, - MakeNotAllowedError && makeNotAllowedError) +ref makeFSInputAccessor(const CanonPath & root) { - return make_ref(root, std::move(allowedPaths), std::move(makeNotAllowedError)); + return make_ref(root); } -ref makeStorePathAccessor( +ref makeStorePathAccessor( ref store, - const StorePath & storePath, - MakeNotAllowedError && makeNotAllowedError) + const StorePath & storePath) { - return makeFSInputAccessor(CanonPath(store->toRealPath(storePath)), {}, std::move(makeNotAllowedError)); + return makeFSInputAccessor(CanonPath(store->toRealPath(storePath))); } SourcePath getUnfilteredRootPath(CanonPath path) diff --git a/src/libfetchers/fs-input-accessor.hh b/src/libfetchers/fs-input-accessor.hh index 19a5211c8..ba5af5887 100644 --- a/src/libfetchers/fs-input-accessor.hh +++ b/src/libfetchers/fs-input-accessor.hh @@ -7,26 +7,12 @@ namespace nix { class StorePath; class Store; -struct FSInputAccessor : InputAccessor -{ - virtual void checkAllowed(const CanonPath & absPath) = 0; +ref makeFSInputAccessor( + const CanonPath & root); - virtual void allowPath(CanonPath path) = 0; - - virtual bool hasAccessControl() = 0; -}; - -typedef std::function MakeNotAllowedError; - -ref makeFSInputAccessor( - const CanonPath & root, - std::optional> && allowedPaths = {}, - MakeNotAllowedError && makeNotAllowedError = {}); - -ref makeStorePathAccessor( +ref makeStorePathAccessor( ref store, - const StorePath & storePath, - MakeNotAllowedError && makeNotAllowedError = {}); + const StorePath & storePath); SourcePath getUnfilteredRootPath(CanonPath path); diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 8cd74057c..ff4b1e823 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -9,6 +9,7 @@ #include "processes.hh" #include "git.hh" #include "fs-input-accessor.hh" +#include "filtering-input-accessor.hh" #include "mounted-input-accessor.hh" #include "git-utils.hh" #include "logging.hh" @@ -639,7 +640,10 @@ struct GitInputScheme : InputScheme repoInfo.workdirInfo.files.insert(submodule.path); ref accessor = - makeFSInputAccessor(CanonPath(repoInfo.url), repoInfo.workdirInfo.files, makeNotAllowedError(repoInfo.url)); + AllowListInputAccessor::create( + makeFSInputAccessor(CanonPath(repoInfo.url)), + std::move(repoInfo.workdirInfo.files), + makeNotAllowedError(repoInfo.url)); /* If the repo has submodules, return a mounted input accessor consisting of the accessor for the top-level repo and the From cab41025d85d3b02f5175cf7ca2611c7a44c2cdd Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 30 Nov 2023 23:04:05 +0100 Subject: [PATCH 021/141] mention renaming of `nix doctor` --- doc/manual/rl-next/nix-config-show.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/rl-next/nix-config-show.md b/doc/manual/rl-next/nix-config-show.md index 08ad207cb..b2ad3c666 100644 --- a/doc/manual/rl-next/nix-config-show.md +++ b/doc/manual/rl-next/nix-config-show.md @@ -3,6 +3,6 @@ issues: #7672 prs: #9477 description: { -`nix show-config` was renamed to `nix config show` to be more consistent with the rest of the command-line interface. +`nix show-config` was renamed to `nix config show`, and `nix doctor` was renamed to `nix config check`, to be more consistent with the rest of the command-line interface. } From 39de819edaef2dc3e308a490bbb2b1622a932771 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 9 Oct 2023 10:08:22 +0200 Subject: [PATCH 022/141] rename debugging helper environment variable --- src/libutil/error.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 8488e7e21..72c346cb5 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -159,11 +159,11 @@ static std::string indent(std::string_view indentFirst, std::string_view indentR /** * A development aid for finding missing positions, to improve error messages. Example use: * - * NIX_DEVELOPER_SHOW_UNKNOWN_LOCATIONS=1 _NIX_TEST_ACCEPT=1 make tests/lang.sh.test + * _NIX_EVAL_SHOW_UNKNOWN_LOCATIONS=1 _NIX_TEST_ACCEPT=1 make tests/lang.sh.test * git diff -U20 tests * */ -static bool printUnknownLocations = getEnv("_NIX_DEVELOPER_SHOW_UNKNOWN_LOCATIONS").has_value(); +static bool printUnknownLocations = getEnv("_NIX_EVAL_SHOW_UNKNOWN_LOCATIONS").has_value(); /** * Print a position, if it is known. From 0301b8fc7354a94dc03b57f796bfa6e853758af8 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 30 Nov 2023 21:59:08 +0100 Subject: [PATCH 023/141] reword the experimental feature notice - put the highlight box around all the relevant instructions - simplify the wording - make the link more prominent by using the whole phrase for the link text --- doc/manual/generate-settings.nix | 21 ++++++------- doc/manual/generate-store-info.nix | 48 +++++++++++++++--------------- 2 files changed, 35 insertions(+), 34 deletions(-) diff --git a/doc/manual/generate-settings.nix b/doc/manual/generate-settings.nix index 3add10075..74446b70b 100644 --- a/doc/manual/generate-settings.nix +++ b/doc/manual/generate-settings.nix @@ -31,18 +31,19 @@ let experimentalFeatureNote = optionalString (experimentalFeature != null) '' > **Warning** + > > This setting is part of an > [experimental feature](@docroot@/contributing/experimental-features.md). - - To change this setting, you need to make sure the corresponding experimental feature, - [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}), - is enabled. - For example, include the following in [`nix.conf`](#): - - ``` - extra-experimental-features = ${experimentalFeature} - ${setting} = ... - ``` + > + > To change this setting, make sure the + > [`${experimentalFeature}` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}) + > is enabled. + > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): + > + > ``` + > extra-experimental-features = ${experimentalFeature} + > ${setting} = ... + > ``` ''; showDefault = documentDefault: defaultValue: diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix index 73defcb71..8e26edb65 100644 --- a/doc/manual/generate-store-info.nix +++ b/doc/manual/generate-store-info.nix @@ -1,6 +1,6 @@ let inherit (builtins) attrValues mapAttrs; - inherit (import ) concatStrings optionalString; + inherit (import ) concatStrings optionalString squash; showSettings = import ; in @@ -10,36 +10,36 @@ let showStore = name: { settings, doc, experimentalFeature }: let + result = squash '' + # ${name} - result = '' - ## ${name} + ${doc} - ${doc} + ${experimentalFeatureNote} - ${experimentalFeatureNote} + ## Settings - ### Settings - - ${showSettings { prefix = "store-${slug}"; inherit inlineHTML; } settings} - ''; + ${showSettings { prefix = "store-${slug}"; inherit inlineHTML; } settings} + ''; # markdown doesn't like spaces in URLs slug = builtins.replaceStrings [ " " ] [ "-" ] name; - experimentalFeatureNote = optionalString (experimentalFeature != null) '' - > **Warning** - > This store is part of an - > [experimental feature](@docroot@/contributing/experimental-features.md). - - To use this store, you need to make sure the corresponding experimental feature, - [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}), - is enabled. - For example, include the following in [`nix.conf`](#): - - ``` - extra-experimental-features = ${experimentalFeature} - ``` - ''; - in result; + experimentalFeatureNote = optionalString (experimentalFeature != null) '' + > **Warning** + > + > This store is part of an + > [experimental feature](@docroot@/contributing/experimental-features.md). + > + > To use this store, make sure the + > [`${experimentalFeature}` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}) + > is enabled. + > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): + > + > ``` + > extra-experimental-features = ${experimentalFeature} + > ``` + ''; + in result; in concatStrings (attrValues (mapAttrs showStore storesInfo)) From c982198485a995d40b01b8caf62df5458046614d Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Thu, 30 Nov 2023 22:48:44 +0000 Subject: [PATCH 024/141] First step --- binary-tarball.nix | 81 ++++++++++++++ flake.nix | 262 +++++++++++++-------------------------------- lowdown.nix | 22 ++++ package.nix | 239 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 415 insertions(+), 189 deletions(-) create mode 100644 binary-tarball.nix create mode 100644 lowdown.nix create mode 100644 package.nix diff --git a/binary-tarball.nix b/binary-tarball.nix new file mode 100644 index 000000000..1fa185519 --- /dev/null +++ b/binary-tarball.nix @@ -0,0 +1,81 @@ +{ runCommand +, version +, system +, nix +, cacert +}: + +let + + installerClosureInfo = buildPackages.closureInfo { + rootPaths = [ nix cacert ]; + }; + + env = { + meta.description = "Distribution-independent Nix bootstrap binaries for ${system}"; + }; + +in + +runCommand "nix-binary-tarball-${version}" env '' + cp ${installerClosureInfo}/registration $TMPDIR/reginfo + cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh + substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \ + --subst-var-by nix ${nix} \ + --subst-var-by cacert ${cacert} + + substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \ + --subst-var-by nix ${nix} \ + --subst-var-by cacert ${cacert} + substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \ + --subst-var-by nix ${nix} \ + --subst-var-by cacert ${cacert} + substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \ + --subst-var-by nix ${nix} \ + --subst-var-by cacert ${cacert} + + if type -p shellcheck; then + # SC1090: Don't worry about not being able to find + # $nix/etc/profile.d/nix.sh + shellcheck --exclude SC1090 $TMPDIR/install + shellcheck $TMPDIR/create-darwin-volume.sh + shellcheck $TMPDIR/install-darwin-multi-user.sh + shellcheck $TMPDIR/install-systemd-multi-user.sh + + # SC1091: Don't panic about not being able to source + # /etc/profile + # SC2002: Ignore "useless cat" "error", when loading + # .reginfo, as the cat is a much cleaner + # implementation, even though it is "useless" + # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving + # root's home directory + shellcheck --external-sources \ + --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user + fi + + chmod +x $TMPDIR/install + chmod +x $TMPDIR/create-darwin-volume.sh + chmod +x $TMPDIR/install-darwin-multi-user.sh + chmod +x $TMPDIR/install-systemd-multi-user.sh + chmod +x $TMPDIR/install-multi-user + dir=nix-${version}-${system} + fn=$out/$dir.tar.xz + mkdir -p $out/nix-support + echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products + tar cvfJ $fn \ + --owner=0 --group=0 --mode=u+rw,uga+r \ + --mtime='1970-01-01' \ + --absolute-names \ + --hard-dereference \ + --transform "s,$TMPDIR/install,$dir/install," \ + --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \ + --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \ + --transform "s,$NIX_STORE,$dir/store,S" \ + $TMPDIR/install \ + $TMPDIR/create-darwin-volume.sh \ + $TMPDIR/install-darwin-multi-user.sh \ + $TMPDIR/install-systemd-multi-user.sh \ + $TMPDIR/install-multi-user \ + $TMPDIR/reginfo \ + $(cat ${installerClosureInfo}/store-paths) +'' diff --git a/flake.nix b/flake.nix index 33673575b..a1fc1cd1c 100644 --- a/flake.nix +++ b/flake.nix @@ -7,7 +7,7 @@ inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; }; - outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src, flake-compat, libgit2 }: + outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src, libgit2 }: let inherit (nixpkgs) lib; @@ -34,7 +34,14 @@ "x86_64-freebsd13" "x86_64-netbsd" ]; - stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" "libcxxStdenv" "ccacheStdenv" ]; + stdenvs = [ + "ccacheStdenv" + "clang11Stdenv" + "clangStdenv" + "gccStdenv" + "libcxxStdenv" + "stdenv" + ]; forAllSystems = lib.genAttrs systems; @@ -326,82 +333,18 @@ ''; }; - binaryTarball = nix: pkgs: - let - inherit (pkgs) buildPackages; - inherit (pkgs) cacert; - installerClosureInfo = buildPackages.closureInfo { rootPaths = [ nix cacert ]; }; - in - - buildPackages.runCommand "nix-binary-tarball-${version}" - { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck; - meta.description = "Distribution-independent Nix bootstrap binaries for ${pkgs.system}"; - } - '' - cp ${installerClosureInfo}/registration $TMPDIR/reginfo - cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh - substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \ - --subst-var-by nix ${nix} \ - --subst-var-by cacert ${cacert} - - substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \ - --subst-var-by nix ${nix} \ - --subst-var-by cacert ${cacert} - substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \ - --subst-var-by nix ${nix} \ - --subst-var-by cacert ${cacert} - substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \ - --subst-var-by nix ${nix} \ - --subst-var-by cacert ${cacert} - - if type -p shellcheck; then - # SC1090: Don't worry about not being able to find - # $nix/etc/profile.d/nix.sh - shellcheck --exclude SC1090 $TMPDIR/install - shellcheck $TMPDIR/create-darwin-volume.sh - shellcheck $TMPDIR/install-darwin-multi-user.sh - shellcheck $TMPDIR/install-systemd-multi-user.sh - - # SC1091: Don't panic about not being able to source - # /etc/profile - # SC2002: Ignore "useless cat" "error", when loading - # .reginfo, as the cat is a much cleaner - # implementation, even though it is "useless" - # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving - # root's home directory - shellcheck --external-sources \ - --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user - fi - - chmod +x $TMPDIR/install - chmod +x $TMPDIR/create-darwin-volume.sh - chmod +x $TMPDIR/install-darwin-multi-user.sh - chmod +x $TMPDIR/install-systemd-multi-user.sh - chmod +x $TMPDIR/install-multi-user - dir=nix-${version}-${pkgs.system} - fn=$out/$dir.tar.xz - mkdir -p $out/nix-support - echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products - tar cvfJ $fn \ - --owner=0 --group=0 --mode=u+rw,uga+r \ - --mtime='1970-01-01' \ - --absolute-names \ - --hard-dereference \ - --transform "s,$TMPDIR/install,$dir/install," \ - --transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \ - --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \ - --transform "s,$NIX_STORE,$dir/store,S" \ - $TMPDIR/install \ - $TMPDIR/create-darwin-volume.sh \ - $TMPDIR/install-darwin-multi-user.sh \ - $TMPDIR/install-systemd-multi-user.sh \ - $TMPDIR/install-multi-user \ - $TMPDIR/reginfo \ - $(cat ${installerClosureInfo}/store-paths) - ''; + binaryTarball = nix: pkgs: pkgs.callPackage ./binary-tarball.nix { + inherit nix; + }; overlayFor = getStdenv: final: prev: - let currentStdenv = getStdenv final; in + let + stdenv = getStdenv final; + + lowdown-nix = final.callPackage ./lowdown.nix { + inherit lowdown-src stdenv; + }; + in { nixStable = prev.nix; @@ -409,129 +352,70 @@ nixUnstable = prev.nixUnstable; nix = - with final; - with commonDeps { - inherit pkgs; - inherit (currentStdenv.hostPlatform) isStatic; - }; - let - canRunInstalled = currentStdenv.buildPlatform.canExecute currentStdenv.hostPlatform; - in currentStdenv.mkDerivation (finalAttrs: { - name = "nix-${version}"; - inherit version; + let + officialRelease = false; + versionSuffix = + if officialRelease + then "" + else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}"; - src = nixSrc; - VERSION_SUFFIX = versionSuffix; + sh = final.busybox-sandbox-shell or (final.busybox.override { + useMusl = true; + enableStatic = true; + enableMinimal = true; + extraConfig = '' + CONFIG_FEATURE_FANCY_ECHO y + CONFIG_FEATURE_SH_MATH y + CONFIG_FEATURE_SH_MATH_64 y - outputs = [ "out" "dev" "doc" ] - ++ lib.optional (currentStdenv.hostPlatform != currentStdenv.buildPlatform) "check"; + CONFIG_ASH y + CONFIG_ASH_OPTIMIZE_FOR_SIZE y - nativeBuildInputs = nativeBuildDeps; - buildInputs = buildDeps - # There have been issues building these dependencies - ++ lib.optionals (currentStdenv.hostPlatform == currentStdenv.buildPlatform) awsDeps - ++ lib.optionals finalAttrs.doCheck checkDeps; + CONFIG_ASH_ALIAS y + CONFIG_ASH_BASH_COMPAT y + CONFIG_ASH_CMDCMD y + CONFIG_ASH_ECHO y + CONFIG_ASH_GETOPTS y + CONFIG_ASH_INTERNAL_GLOB y + CONFIG_ASH_JOB_CONTROL y + CONFIG_ASH_PRINTF y + CONFIG_ASH_TEST y + ''; + }); - propagatedBuildInputs = propagatedDeps; + boehmgc = (final.boehmgc.override { + enableLargeConfig = true; + }).overrideAttrs(o: { + patches = (o.patches or []) ++ [ + ./boehmgc-coroutine-sp-fallback.diff - disallowedReferences = [ boost-nix ]; + # https://github.com/ivmai/bdwgc/pull/586 + ./boehmgc-traceable_allocator-public.diff + ]; + }); - preConfigure = lib.optionalString (! currentStdenv.hostPlatform.isStatic) - '' - # Copy libboost_context so we don't get all of Boost in our closure. - # https://github.com/NixOS/nixpkgs/issues/45462 - mkdir -p $out/lib - cp -pd ${boost-nix}/lib/{libboost_context*,libboost_thread*,libboost_system*,libboost_regex*} $out/lib - rm -f $out/lib/*.a - ${lib.optionalString currentStdenv.hostPlatform.isLinux '' - chmod u+w $out/lib/*.so.* - patchelf --set-rpath $out/lib:${currentStdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* - ''} - ${lib.optionalString currentStdenv.hostPlatform.isDarwin '' - for LIB in $out/lib/*.dylib; do - chmod u+w $LIB - install_name_tool -id $LIB $LIB - install_name_tool -delete_rpath ${boost-nix}/lib/ $LIB || true - done - install_name_tool -change ${boost-nix}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib - ''} - ''; - - configureFlags = configureFlags ++ - [ "--sysconfdir=/etc" ] ++ - lib.optional stdenv.hostPlatform.isStatic "--enable-embedded-sandbox-shell" ++ - [ (lib.enableFeature finalAttrs.doCheck "tests") ] ++ - lib.optionals finalAttrs.doCheck testConfigureFlags ++ - lib.optional (!canRunInstalled) "--disable-doc-gen"; - - enableParallelBuilding = true; - - makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1"; - - doCheck = true; - - installFlags = "sysconfdir=$(out)/etc"; - - postInstall = '' - mkdir -p $doc/nix-support - echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products - ${lib.optionalString currentStdenv.hostPlatform.isStatic '' - mkdir -p $out/nix-support - echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products - ''} - ${lib.optionalString currentStdenv.isDarwin '' - install_name_tool \ - -change ${boost-nix}/lib/libboost_context.dylib \ - $out/lib/libboost_context.dylib \ - $out/lib/libnixutil.dylib - install_name_tool \ - -change ${boost-nix}/lib/libboost_regex.dylib \ - $out/lib/libboost_regex.dylib \ - $out/lib/libnixexpr.dylib - ''} - ''; - - doInstallCheck = finalAttrs.doCheck; - installCheckFlags = "sysconfdir=$(out)/etc"; - installCheckTarget = "installcheck"; # work around buggy detection in stdenv - - separateDebugInfo = !currentStdenv.hostPlatform.isStatic; - - strictDeps = true; - - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - - passthru.perl-bindings = final.callPackage ./perl { - inherit fileset; - stdenv = currentStdenv; + in final.callPackage ./package.nix { + inherit + boehmgc + fileset + sh + stdenv + versionSuffix + ; + boost = final.boost.override { enableIcu = false; }; + libgit2 = final.libgit2.overrideAttrs (attrs: { + src = libgit2; + version = libgit2.lastModifiedDate; + cmakeFlags = attrs.cmakeFlags or [] + ++ [ "-DUSE_SSH=exec" ]; + }); + lowdown = lowdown-nix; + officialRelease = false; }; - meta.platforms = lib.platforms.unix; - meta.mainProgram = "nix"; - }); - - boost-nix = final.boost.override { - enableIcu = false; + inherit lowdown-nix; }; - lowdown-nix = with final; currentStdenv.mkDerivation rec { - name = "lowdown-0.9.0"; - - src = lowdown-src; - - outputs = [ "out" "bin" "dev" ]; - - nativeBuildInputs = [ buildPackages.which ]; - - configurePhase = '' - ${if (currentStdenv.isDarwin && currentStdenv.isAarch64) then "echo \"HAVE_SANDBOX_INIT=false\" > configure.local" else ""} - ./configure \ - PREFIX=${placeholder "dev"} \ - BINDIR=${placeholder "bin"}/bin - ''; - }; - }; - in { # A Nixpkgs overlay that overrides the 'nix' and # 'nix.perl-bindings' packages. diff --git a/lowdown.nix b/lowdown.nix new file mode 100644 index 000000000..5f469fad5 --- /dev/null +++ b/lowdown.nix @@ -0,0 +1,22 @@ +{ lib +, stdenv +, which +, lowdown-src +}: + +stdenv.mkDerivation rec { + name = "lowdown-0.9.0"; + + src = lowdown-src; + + outputs = [ "out" "bin" "dev" ]; + + nativeBuildInputs = [ which ]; + + configurePhase = '' + ${lib.optionalString (stdenv.isDarwin && stdenv.isAarch64) "echo \"HAVE_SANDBOX_INIT=false\" > configure.local"} + ./configure \ + PREFIX=${placeholder "dev"} \ + BINDIR=${placeholder "bin"}/bin + ''; +} diff --git a/package.nix b/package.nix new file mode 100644 index 000000000..ae075acf7 --- /dev/null +++ b/package.nix @@ -0,0 +1,239 @@ +{ lib +, callPackage +, stdenv +, versionSuffix ? "" +, officialRelease ? false +, buildUnreleasedNotes ? false +, autoconf-archive +, autoreconfHook +, aws-sdk-cpp +, boehmgc +, nlohmann_json +, bison +, boost +, brotli +, bzip2 +, changelog-d +, curl +, editline +, fileset +, flex +, git +, gtest +, jq +, libarchive +, libcpuid +, libgit2 +, libseccomp +, libsodium +, lowdown +, mdbook +, mdbook-linkcheck +, mercurial +, openssh +, openssl +, pkg-config +, rapidcheck +, sh +, sqlite +, util-linux +, xz +}: + +let + + version = lib.fileContents ./.version + versionSuffix; + + inherit (stdenv.hostPlatform) isStatic; + + canRunInstalled = stdenv.buildPlatform.canExecute stdenv.hostPlatform; +in + +stdenv.mkDerivation (finalAttrs: { + name = "nix-${version}"; + inherit version; + + src = + let + baseFiles = fileset.fileFilter (f: f.name != ".gitignore") ./.; + configureFiles = fileset.unions [ + ./.version + ./configure.ac + ./m4 + # TODO: do we really need README.md? It doesn't seem used in the build. + ./README.md + ]; + + topLevelBuildFiles = fileset.unions [ + ./local.mk + ./Makefile + ./Makefile.config.in + ./mk + ]; + + functionalTestFiles = fileset.unions [ + ./tests/functional + (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) + ]; + + in + fileset.toSource { + root = ./.; + fileset = fileset.intersect baseFiles (fileset.unions [ + configureFiles + topLevelBuildFiles + ./boehmgc-coroutine-sp-fallback.diff + ./doc + ./misc + ./precompiled-headers.h + ./src + ./unit-test-data + ./COPYING + ./scripts/local.mk + functionalTestFiles + ]); + }; + + VERSION_SUFFIX = versionSuffix; + + outputs = [ "out" "dev" "doc" ] + ++ lib.optional (stdenv.hostPlatform != stdenv.buildPlatform) "check"; + + nativeBuildInputs = [ + bison + flex + (lib.getBin lowdown) + mdbook + mdbook-linkcheck + autoconf-archive + autoreconfHook + pkg-config + + # Tests + git + mercurial # FIXME: remove? only needed for tests + jq # Also for custom mdBook preprocessor. + openssh # only needed for tests (ssh-keygen) + ] + ++ lib.optional stdenv.hostPlatform.isLinux util-linux + # Official releases don't have rl-next, so we don't need to compile a changelog + ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d; + + buildInputs = [ + boost + brotli + bzip2 + curl + editline + libarchive + libgit2 + libsodium + lowdown + openssl + sqlite + xz + ] + ++ lib.optionals stdenv.isLinux [libseccomp] + ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid + # There have been issues building these dependencies + ++ lib.optionals (stdenv.hostPlatform == stdenv.buildPlatform) (lib.optional (stdenv.isLinux || stdenv.isDarwin) + (aws-sdk-cpp.override { + apis = ["s3" "transfer"]; + customMemoryManagement = false; + })) + ++ lib.optionals finalAttrs.doCheck ([ + gtest + rapidcheck + ]); + + propagatedBuildInputs = [ + boehmgc + nlohmann_json + ]; + + disallowedReferences = [ boost ]; + + preConfigure = lib.optionalString (! stdenv.hostPlatform.isStatic) + '' + # Copy libboost_context so we don't get all of Boost in our closure. + # https://github.com/NixOS/nixpkgs/issues/45462 + mkdir -p $out/lib + cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*,libboost_regex*} $out/lib + rm -f $out/lib/*.a + ${lib.optionalString stdenv.hostPlatform.isLinux '' + chmod u+w $out/lib/*.so.* + patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* + ''} + ${lib.optionalString stdenv.hostPlatform.isDarwin '' + for LIB in $out/lib/*.dylib; do + chmod u+w $LIB + install_name_tool -id $LIB $LIB + install_name_tool -delete_rpath ${boost}/lib/ $LIB || true + done + install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib + ''} + ''; + + configureFlags = + lib.optionals stdenv.isLinux [ + "--with-boost=${boost}/lib" + "--with-sandbox-shell=${sh}/bin/busybox" + ] + ++ lib.optionals (stdenv.isLinux && !(isStatic && stdenv.system == "aarch64-linux")) [ + "LDFLAGS=-fuse-ld=gold" + ] + ++ [ "--sysconfdir=/etc" ] + ++ lib.optional stdenv.hostPlatform.isStatic "--enable-embedded-sandbox-shell" + ++ [ (lib.enableFeature finalAttrs.doCheck "tests") ] + ++ lib.optionals finalAttrs.doCheck ([ "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include" ] + ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ + "--enable-install-unit-tests" + "--with-check-bin-dir=${builtins.placeholder "check"}/bin" + "--with-check-lib-dir=${builtins.placeholder "check"}/lib" + ]) + ++ lib.optional (!canRunInstalled) "--disable-doc-gen"; + + enableParallelBuilding = true; + + makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1"; + + doCheck = true; + + installFlags = "sysconfdir=$(out)/etc"; + + postInstall = '' + mkdir -p $doc/nix-support + echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products + ${lib.optionalString stdenv.hostPlatform.isStatic '' + mkdir -p $out/nix-support + echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products + ''} + ${lib.optionalString stdenv.isDarwin '' + install_name_tool \ + -change ${boost}/lib/libboost_context.dylib \ + $out/lib/libboost_context.dylib \ + $out/lib/libnixutil.dylib + install_name_tool \ + -change ${boost}/lib/libboost_regex.dylib \ + $out/lib/libboost_regex.dylib \ + $out/lib/libnixexpr.dylib + ''} + ''; + + doInstallCheck = finalAttrs.doCheck; + installCheckFlags = "sysconfdir=$(out)/etc"; + installCheckTarget = "installcheck"; # work around buggy detection in stdenv + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + strictDeps = true; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + passthru.perl-bindings = callPackage ./perl { + inherit fileset stdenv; + }; + + meta.platforms = lib.platforms.unix; + meta.mainProgram = "nix"; +}) From c64190e65048547712fcf7a0ae09fbfd0a709474 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Thu, 30 Nov 2023 22:49:02 +0000 Subject: [PATCH 025/141] Run statix --- flake.nix | 6 +++--- package.nix | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flake.nix b/flake.nix index a1fc1cd1c..e32a84ae5 100644 --- a/flake.nix +++ b/flake.nix @@ -7,7 +7,7 @@ inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; }; - outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src, libgit2 }: + outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src, libgit2, ... }: let inherit (nixpkgs) lib; @@ -183,7 +183,7 @@ "--enable-internal-api-docs" ]; - changelog-d = pkgs.buildPackages.changelog-d; + inherit (pkgs.buildPackages) changelog-d; nativeBuildDeps = [ @@ -349,7 +349,7 @@ nixStable = prev.nix; # Forward from the previous stage as we don’t want it to pick the lowdown override - nixUnstable = prev.nixUnstable; + inherit (prev) nixUnstable; nix = let diff --git a/package.nix b/package.nix index ae075acf7..8d62120fb 100644 --- a/package.nix +++ b/package.nix @@ -141,10 +141,10 @@ stdenv.mkDerivation (finalAttrs: { apis = ["s3" "transfer"]; customMemoryManagement = false; })) - ++ lib.optionals finalAttrs.doCheck ([ + ++ lib.optionals finalAttrs.doCheck [ gtest rapidcheck - ]); + ]; propagatedBuildInputs = [ boehmgc From f55ee7cf7753caee7a27052fab679d8c8fe27cc4 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Thu, 30 Nov 2023 22:53:07 +0000 Subject: [PATCH 026/141] little refactoring --- flake.nix | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index e32a84ae5..544a07ba6 100644 --- a/flake.nix +++ b/flake.nix @@ -459,8 +459,21 @@ # to https://nixos.org/nix/install. It downloads the binary # tarball for the user's system and calls the second half of the # installation script. - installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" "armv6l-linux" "armv7l-linux" ]; - installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" "armv6l-linux" "armv7l-linux"]; + installerScript = installScriptFor [ + "aarch64-linux" + "armv6l-linux" + "armv7l-linux" + "i686-linux" + "x86_64-linux" + "aarch64-darwin" + "x86_64-darwin" + ]; + installerScriptForGHA = installScriptFor [ + "armv6l-linux" + "armv7l-linux" + "x86_64-linux" + "x86_64-darwin" + ]; # docker image with Nix inside dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); From 02d9cf2d303e4e7e283dba2f3181f3e40843c354 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 1 Dec 2023 00:41:19 +0100 Subject: [PATCH 027/141] shorten the quick start chapter this focuses on `nix-shell -p` and refers to search.nixos.org for package search, which is currently the easiest and most effective way to find program names. --- doc/manual/src/quick-start.md | 87 +++++++---------------------------- 1 file changed, 16 insertions(+), 71 deletions(-) diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md index 5f54abbde..04a0b7c96 100644 --- a/doc/manual/src/quick-start.md +++ b/doc/manual/src/quick-start.md @@ -1,10 +1,9 @@ # Quick Start -This chapter is for impatient people who don't like reading -documentation. For more in-depth information you are kindly referred -to subsequent chapters. +This chapter is for impatient people who don't like reading documentation. +For more in-depth information you are kindly referred to subsequent chapters. -1. Install Nix by running the following: +1. Install Nix: ```console $ curl -L https://nixos.org/nix/install | sh @@ -13,87 +12,33 @@ to subsequent chapters. The install script will use `sudo`, so make sure you have sufficient rights. On Linux, `--daemon` can be omitted for a single-user install. - For other installation methods, see [here](installation/index.md). + For other installation methods, see the detailed [installation instructions](installation/index.md). -1. See what installable packages are currently available in the - channel: +1. Run software without installing it permanently: ```console - $ nix-env --query --available --attr-path - nixpkgs.docbook_xml_dtd_43 docbook-xml-4.3 - nixpkgs.docbook_xml_dtd_45 docbook-xml-4.5 - nixpkgs.firefox firefox-33.0.2 - nixpkgs.hello hello-2.9 - nixpkgs.libxslt libxslt-1.1.28 - … + $ nix-shell --packages cowsay lolcat ``` -1. Install some packages from the channel: + This downloads the specified packages with all their dependencies, and drops you into a Bash shell where the commands provided by those packages are present. + This will not affect your normal environment: ```console - $ nix-env --install --attr nixpkgs.hello + [nix-shell:~]$ cowsay Hello, Nix! | lolcat ``` - This should download pre-built packages; it should not build them - locally (if it does, something went wrong). - -1. Test that they work: + Exiting the shell will make the programs disappear again: ```console - $ which hello - /home/eelco/.nix-profile/bin/hello - $ hello - Hello, world! - ``` - -1. Uninstall a package: - - ```console - $ nix-env --uninstall hello - ``` - -1. You can also test a package without installing it: - - ```console - $ nix-shell --packages hello - ``` - - This builds or downloads GNU Hello and its dependencies, then drops - you into a Bash shell where the `hello` command is present, all - without affecting your normal environment: - - ```console - [nix-shell:~]$ hello - Hello, world! - [nix-shell:~]$ exit - - $ hello - hello: command not found + $ lolcat + lolcat: command not found ``` -1. To keep up-to-date with the channel, do: +1. Search for more packages on to try them out. + +1. Free up storage space: ```console - $ nix-channel --update nixpkgs - $ nix-env --upgrade '*' - ``` - - The latter command will upgrade each installed package for which - there is a “newer” version (as determined by comparing the version - numbers). - -1. If you're unhappy with the result of a `nix-env` action (e.g., an - upgraded package turned out not to work properly), you can go back: - - ```console - $ nix-env --rollback - ``` - -1. You should periodically run the Nix garbage collector to get rid of - unused packages, since uninstalls or upgrades don't actually delete - them: - - ```console - $ nix-collect-garbage --delete-old + $ nix-collect-garbage ``` From 908a011a4a2fe4e494e5b6e4c94f013f159f3616 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 1 Dec 2023 00:50:20 +0100 Subject: [PATCH 028/141] Revert "Switch from std::regex to boost::regex" --- flake.nix | 22 +++++++--------------- src/libexpr/local.mk | 2 +- src/libexpr/primops.cc | 35 ++++++++++++----------------------- 3 files changed, 20 insertions(+), 39 deletions(-) diff --git a/flake.nix b/flake.nix index 33673575b..822b3d31e 100644 --- a/flake.nix +++ b/flake.nix @@ -157,7 +157,7 @@ configureFlags = lib.optionals stdenv.isLinux [ - "--with-boost=${boost-nix}/lib" + "--with-boost=${boost}/lib" "--with-sandbox-shell=${sh}/bin/busybox" ] ++ lib.optionals (stdenv.isLinux && !(isStatic && stdenv.system == "aarch64-linux")) [ @@ -210,7 +210,7 @@ version = libgit2.lastModifiedDate; cmakeFlags = (attrs.cmakeFlags or []) ++ ["-DUSE_SSH=exec"]; })) - boost-nix + boost lowdown-nix libsodium ] @@ -434,14 +434,14 @@ propagatedBuildInputs = propagatedDeps; - disallowedReferences = [ boost-nix ]; + disallowedReferences = [ boost ]; preConfigure = lib.optionalString (! currentStdenv.hostPlatform.isStatic) '' # Copy libboost_context so we don't get all of Boost in our closure. # https://github.com/NixOS/nixpkgs/issues/45462 mkdir -p $out/lib - cp -pd ${boost-nix}/lib/{libboost_context*,libboost_thread*,libboost_system*,libboost_regex*} $out/lib + cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib rm -f $out/lib/*.a ${lib.optionalString currentStdenv.hostPlatform.isLinux '' chmod u+w $out/lib/*.so.* @@ -451,9 +451,9 @@ for LIB in $out/lib/*.dylib; do chmod u+w $LIB install_name_tool -id $LIB $LIB - install_name_tool -delete_rpath ${boost-nix}/lib/ $LIB || true + install_name_tool -delete_rpath ${boost}/lib/ $LIB || true done - install_name_tool -change ${boost-nix}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib + install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib ''} ''; @@ -481,13 +481,9 @@ ''} ${lib.optionalString currentStdenv.isDarwin '' install_name_tool \ - -change ${boost-nix}/lib/libboost_context.dylib \ + -change ${boost}/lib/libboost_context.dylib \ $out/lib/libboost_context.dylib \ $out/lib/libnixutil.dylib - install_name_tool \ - -change ${boost-nix}/lib/libboost_regex.dylib \ - $out/lib/libboost_regex.dylib \ - $out/lib/libnixexpr.dylib ''} ''; @@ -510,10 +506,6 @@ meta.mainProgram = "nix"; }); - boost-nix = final.boost.override { - enableIcu = false; - }; - lowdown-nix = with final; currentStdenv.mkDerivation rec { name = "lowdown-0.9.0"; diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index c07a18bb5..ed6bc761a 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -16,7 +16,7 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib libexpr_LIBS = libutil libstore libfetchers -libexpr_LDFLAGS += -lboost_context -lboost_regex -pthread +libexpr_LDFLAGS += -lboost_context -pthread ifdef HOST_LINUX libexpr_LDFLAGS += -ldl endif diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index ebf2549e4..146a7603c 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -17,7 +17,6 @@ #include "primops.hh" #include -#include #include #include @@ -26,6 +25,7 @@ #include #include +#include #include #include @@ -3886,30 +3886,19 @@ static RegisterPrimOp primop_convertHash({ .fun = prim_convertHash, }); -// regex aliases, switch between boost and std -using regex = boost::regex; -using regex_error = boost::regex_error; -using cmatch = boost::cmatch; -using cregex_iterator = boost::cregex_iterator; -namespace regex_constants = boost::regex_constants; -// overloaded function alias -constexpr auto regex_match = [] (auto &&...args) { - return boost::regex_match(std::forward(args)...); - }; - struct RegexCache { // TODO use C++20 transparent comparison when available - std::unordered_map cache; + std::unordered_map cache; std::list keys; - regex get(std::string_view re) + std::regex get(std::string_view re) { auto it = cache.find(re); if (it != cache.end()) return it->second; keys.emplace_back(re); - return cache.emplace(keys.back(), regex(keys.back(), regex::extended)).first->second; + return cache.emplace(keys.back(), std::regex(keys.back(), std::regex::extended)).first->second; } }; @@ -3929,8 +3918,8 @@ void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v) NixStringContext context; const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.match"); - cmatch match; - if (!regex_match(str.begin(), str.end(), match, regex)) { + std::cmatch match; + if (!std::regex_match(str.begin(), str.end(), match, regex)) { v.mkNull(); return; } @@ -3945,8 +3934,8 @@ void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v) (v.listElems()[i] = state.allocValue())->mkString(match[i + 1].str()); } - } catch (regex_error & e) { - if (e.code() == regex_constants::error_space) { + } catch (std::regex_error & e) { + if (e.code() == std::regex_constants::error_space) { // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ state.debugThrowLastTrace(EvalError({ .msg = hintfmt("memory limit exceeded by regular expression '%s'", re), @@ -4009,8 +3998,8 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v) NixStringContext context; const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.split"); - auto begin = cregex_iterator(str.begin(), str.end(), regex); - auto end = cregex_iterator(); + auto begin = std::cregex_iterator(str.begin(), str.end(), regex); + auto end = std::cregex_iterator(); // Any matches results are surrounded by non-matching results. const size_t len = std::distance(begin, end); @@ -4049,8 +4038,8 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v) assert(idx == 2 * len + 1); - } catch (regex_error & e) { - if (e.code() == regex_constants::error_space) { + } catch (std::regex_error & e) { + if (e.code() == std::regex_constants::error_space) { // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ state.debugThrowLastTrace(EvalError({ .msg = hintfmt("memory limit exceeded by regular expression '%s'", re), From 4781e7fa7048d2861172baaaa04a7be4b8a2b631 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 30 Nov 2023 23:07:09 +0100 Subject: [PATCH 029/141] Document each store type on its own page This makes for more useful manual table of contents, that displays the information at a glance. The `nix help-stores` command is kept as-is, even though it will show up in the manual with the same information as these pages due to the way it is written as a "`--help`-style" command. Deciding what to do with that command is left for a later PR. This change also lists all store types at the top of the respective overview page. Co-authored-by: John Ericson `. + let + help-stores = '' + ${index} - maybeOptions = let - allVisibleOptions = filterAttrs - (_: o: ! o.hiddenCategory) - (details.flags // toplevel.flags); - in optionalString (allVisibleOptions != {}) '' - # Options + ${allStores} + ''; + index = replaceStrings + [ "@store-types@" ] [ storesOverview ] + details.doc; + storesOverview = + let + showEntry = store: + "- [${store.name}](#${store.slug})"; + in + concatStringsSep "\n" (map showEntry storesList) + "\n"; + allStores = concatStringsSep "\n" (attrValues storePages); + storePages = listToAttrs + (map (s: { name = s.filename; value = s.page; }) storesList); + storesList = showStoreDocs { + storeInfo = commandInfo.stores; + inherit inlineHTML; + }; + in + optionalString (details ? doc) ( + if match "@store-types@" details.doc != [ ] + then help-stores + else details.doc + ); - ${showOptions inlineHTML allVisibleOptions} + maybeOptions = + let + allVisibleOptions = filterAttrs + (_: o: ! o.hiddenCategory) + (details.flags // toplevel.flags); + in + optionalString (allVisibleOptions != { }) '' + # Options - > **Note** - > - > See [`man nix.conf`](@docroot@/command-ref/conf-file.md#command-line-flags) for overriding configuration settings with command line flags. - ''; + ${showOptions inlineHTML allVisibleOptions} + + > **Note** + > + > See [`man nix.conf`](@docroot@/command-ref/conf-file.md#command-line-flags) for overriding configuration settings with command line flags. + ''; showOptions = inlineHTML: allOptions: let diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix index 8e26edb65..57247a181 100644 --- a/doc/manual/generate-store-info.nix +++ b/doc/manual/generate-store-info.nix @@ -1,14 +1,20 @@ let - inherit (builtins) attrValues mapAttrs; - inherit (import ) concatStrings optionalString squash; + inherit (builtins) attrNames listToAttrs concatStringsSep readFile replaceStrings; + inherit (import ) optionalString filterAttrs trim squash toLower unique indent; showSettings = import ; in -inlineHTML: storesInfo: +{ + # data structure describing all stores and their parameters + storeInfo, + # whether to add inline HTML tags + # `lowdown` does not eat those for one of the output modes + inlineHTML, +}: let - showStore = name: { settings, doc, experimentalFeature }: + showStore = { name, slug }: { settings, doc, experimentalFeature }: let result = squash '' # ${name} @@ -22,9 +28,6 @@ let ${showSettings { prefix = "store-${slug}"; inherit inlineHTML; } settings} ''; - # markdown doesn't like spaces in URLs - slug = builtins.replaceStrings [ " " ] [ "-" ] name; - experimentalFeatureNote = optionalString (experimentalFeature != null) '' > **Warning** > @@ -42,4 +45,13 @@ let ''; in result; -in concatStrings (attrValues (mapAttrs showStore storesInfo)) + storesList = map + (name: rec { + inherit name; + slug = replaceStrings [ " " ] [ "-" ] (toLower name); + filename = "${slug}.md"; + page = showStore { inherit name slug; } storeInfo.${name}; + }) + (attrNames storeInfo); + +in storesList diff --git a/doc/manual/generate-store-types.nix b/doc/manual/generate-store-types.nix new file mode 100644 index 000000000..3b78a0e1b --- /dev/null +++ b/doc/manual/generate-store-types.nix @@ -0,0 +1,39 @@ +let + inherit (builtins) attrNames listToAttrs concatStringsSep readFile replaceStrings; + showSettings = import ; + showStoreDocs = import ; +in + +storeInfo: + +let + storesList = showStoreDocs { + inherit storeInfo; + inlineHTML = true; + }; + + index = + let + showEntry = store: + "- [${store.name}](./${store.filename})"; + in + concatStringsSep "\n" (map showEntry storesList); + + "index.md" = replaceStrings + [ "@store-types@" ] [ index ] + (readFile ./src/store/types/index.md.in); + + tableOfContents = + let + showEntry = store: + " - [${store.name}](store/types/${store.filename})"; + in + concatStringsSep "\n" (map showEntry storesList) + "\n"; + + "SUMMARY.md" = tableOfContents; + + storePages = listToAttrs + (map (s: { name = s.filename; value = s.page; }) storesList); + +in +storePages // { inherit "index.md" "SUMMARY.md"; } diff --git a/doc/manual/generate-xp-features.nix b/doc/manual/generate-xp-features.nix index fc7c7d4cf..0eec0e1da 100644 --- a/doc/manual/generate-xp-features.nix +++ b/doc/manual/generate-xp-features.nix @@ -8,4 +8,6 @@ let ${doc} ''; -in xps: (concatStringsSep "\n" (attrValues (mapAttrs showExperimentalFeature xps))) +in + +xps: (concatStringsSep "\n" (attrValues (mapAttrs showExperimentalFeature xps))) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index fa9db9f02..456000d3d 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -97,10 +97,17 @@ $(d)/nix-profiles.5: $(d)/src/command-ref/files/profiles.md $(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@ @rm $^.tmp -$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md +$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md @cp $< $@ @$(call process-includes,$@,$@) +$(d)/src/store/types: $(d)/nix.json $(d)/utils.nix $(d)/generate-store-info.nix $(d)/generate-store-types.nix $(d)/src/store/types/index.md.in $(doc_nix) + @# FIXME: build out of tree! + @rm -rf $@.tmp + $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-store-types.nix (builtins.fromJSON (builtins.readFile $<)).stores' + @# do not destroy existing contents + @mv $@.tmp/* $@/ + $(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(d)/generate-settings.nix $(d)/generate-store-info.nix $(doc_nix) @rm -rf $@ $@.tmp $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix true (builtins.readFile $<)' @@ -200,7 +207,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli # `@docroot@` is to be preserved for documenting the mechanism # FIXME: maybe contributing guides should live right next to the code # instead of in the manual -$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(d)/src/release-notes/rl-next.md +$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(d)/src/release-notes/rl-next.md $(trace-gen) \ tmp="$$(mktemp -d)"; \ cp -r doc/manual "$$tmp"; \ diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 686d3e8d7..c67ddc6cb 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -20,6 +20,8 @@ - [File System Object](store/file-system-object.md) - [Store Object](store/store-object.md) - [Store Path](store/store-path.md) + - [Store Types](store/types/index.md) +{{#include ./store/types/SUMMARY.md}} - [Nix Language](language/index.md) - [Data Types](language/values.md) - [Language Constructs](language/constructs.md) diff --git a/doc/manual/src/contributing/cli-guideline.md b/doc/manual/src/contributing/cli-guideline.md index e53d2d178..f7e24d96b 100644 --- a/doc/manual/src/contributing/cli-guideline.md +++ b/doc/manual/src/contributing/cli-guideline.md @@ -426,7 +426,7 @@ This leads to the following guidelines: ### Examples -This is bad, because all keys must be assumed to be store implementations: +This is bad, because all keys must be assumed to be store types: ```json { diff --git a/doc/manual/src/store/index.md b/doc/manual/src/store/index.md index 8a5305062..f1e8f1402 100644 --- a/doc/manual/src/store/index.md +++ b/doc/manual/src/store/index.md @@ -2,4 +2,4 @@ The *Nix store* is an abstraction to store immutable file system data (such as software packages) that can have dependencies on other such data. -There are multiple implementations of Nix stores with different capabilities, such as the actual filesystem (`/nix/store`) or binary caches. +There are [multiple types of Nix stores](./types/index.md) with different capabilities, such as the default one on the [local filesystem](./types/local-store.md) (`/nix/store`) or [binary caches](./types/http-binary-cache-store.md). diff --git a/doc/manual/src/store/types/index.md.in b/doc/manual/src/store/types/index.md.in index bb166a1fc..b4db553a2 100644 --- a/doc/manual/src/store/types/index.md.in +++ b/doc/manual/src/store/types/index.md.in @@ -1,4 +1,6 @@ -Nix supports different types of stores. These are described below. +Nix supports different types of stores: + +@store-types@ ## Store URL format @@ -39,4 +41,3 @@ store as follows: * Otherwise, use the [local store](#local-store) `/nix/store`. -@stores@ diff --git a/doc/manual/utils.nix b/doc/manual/utils.nix index 849832b2c..19ff49b64 100644 --- a/doc/manual/utils.nix +++ b/doc/manual/utils.nix @@ -1,5 +1,11 @@ with builtins; +let + lowerChars = stringToCharacters "abcdefghijklmnopqrstuvwxyz"; + upperChars = stringToCharacters "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + stringToCharacters = s: genList (p: substring p 1 s) (stringLength s); +in + rec { splitLines = s: filter (x: !isList x) (split "\n" s); @@ -18,6 +24,8 @@ rec { in if replaced == string then string else replaceStringsRec from to replaced; + toLower = replaceStrings upperChars lowerChars; + squash = replaceStringsRec "\n\n\n" "\n\n"; trim = string: diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 401acc38e..193972272 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -141,7 +141,7 @@ MixEvalArgs::MixEvalArgs() .longName = "eval-store", .description = R"( - The [URL of the Nix store](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + The [URL of the Nix store](@docroot@/store/types/index.md#store-url-format) to use for evaluation, i.e. to store derivations (`.drv` files) and inputs referenced by them. )", .category = category, diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index ebf2549e4..4ecdda55c 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -4460,7 +4460,7 @@ void EvalState::createBaseEnv() .doc = R"( Logical file system location of the [Nix store](@docroot@/glossary.md#gloss-store) currently in use. - This value is determined by the `store` parameter in [Store URLs](@docroot@/command-ref/new-cli/nix3-help-stores.md): + This value is determined by the `store` parameter in [Store URLs](@docroot@/store/types/index.md#store-url-format): ```shell-session $ nix-instantiate --store 'dummy://?store=/blah' --eval --expr builtins.storeDir diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 838d2aba2..38b0d516c 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -117,10 +117,11 @@ public: Setting storeUri{this, getEnv("NIX_REMOTE").value_or("auto"), "store", R"( - The [URL of the Nix store](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + The [URL of the Nix store](@docroot@/store/types/index.md#store-url-format) to use for most operations. - See [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md) - for supported store types and settings. + See the + [Store Types](@docroot@/store/types/index.md) + section of the manual for supported store types and settings. )"}; Setting keepFailed{this, false, "keep-failed", @@ -759,7 +760,7 @@ public: Strings{"https://cache.nixos.org/"}, "substituters", R"( - A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) to be used as substituters, separated by whitespace. + A list of [URLs of Nix stores](@docroot@/store/types/index.md#store-url-format) to be used as substituters, separated by whitespace. A substituter is an additional [store]{@docroot@/glossary.md##gloss-store} from which Nix can obtain [store objects](@docroot@/glossary.md#gloss-store-object) instead of building them. Substituters are tried based on their priority value, which each substituter can set independently. @@ -778,7 +779,7 @@ public: Setting trustedSubstituters{ this, {}, "trusted-substituters", R"( - A list of [Nix store URLs](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format), separated by whitespace. + A list of [Nix store URLs](@docroot@/store/types/index.md#store-url-format), separated by whitespace. These are not used by default, but users of the Nix daemon can enable them by specifying [`substituters`](#conf-substituters). Unprivileged users (those set in only [`allowed-users`](#conf-allowed-users) but not [`trusted-users`](#conf-trusted-users)) can pass as `substituters` only those URLs listed in `trusted-substituters`. diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index f16949f42..3d3919882 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -20,7 +20,7 @@ namespace nix { -/* TODO: Separate these store impls into different files, give them better names */ +/* TODO: Separate these store types into different files, give them better names */ RemoteStore::RemoteStore(const Params & params) : RemoteStoreConfig(params) , Store(params) diff --git a/src/libstore/ssh-store-config.hh b/src/libstore/ssh-store-config.hh index bf55d20cf..4ce4ffc4c 100644 --- a/src/libstore/ssh-store-config.hh +++ b/src/libstore/ssh-store-config.hh @@ -20,7 +20,7 @@ struct CommonSSHStoreConfig : virtual StoreConfig const Setting remoteStore{this, "", "remote-store", R"( - [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + [Store URL](@docroot@/store/types/index.md#store-url-format) to be used on the remote machine. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly). )"}; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 8b6bf9aed..e28baf34e 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -30,7 +30,7 @@ namespace nix { /** - * About the class hierarchy of the store implementations: + * About the class hierarchy of the store types: * * Each store type `Foo` consists of two classes: * @@ -962,7 +962,7 @@ OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * ev * - ‘ssh://[user@]’: A remote Nix store accessed by running * ‘nix-store --serve’ via SSH. * - * You can pass parameters to the store implementation by appending + * You can pass parameters to the store type by appending * ‘?key=value&key=value&...’ to the URI. */ ref openStore(const std::string & uri = settings.storeUri.get(), diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 2418e3f4c..e4bdb8cb3 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -252,7 +252,7 @@ constexpr std::array xpFeatureDetails .tag = Xp::ReadOnlyLocalStore, .name = "read-only-local-store", .description = R"( - Allow the use of the `read-only` parameter in [local store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) URIs. + Allow the use of the `read-only` parameter in [local store](@docroot@/store/types/local-store.md) URIs. )", }, { diff --git a/src/nix/nix.md b/src/nix/nix.md index eb150f03b..749456014 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -235,10 +235,14 @@ operate are determined as follows: # Nix stores -Most `nix` subcommands operate on a *Nix store*. These are documented -in [`nix help-stores`](./nix3-help-stores.md). +Most `nix` subcommands operate on a *Nix store*. +The various store types are documented in the +[Store Types](@docroot@/store/types/index.md) +section of the manual. -# Shebang interpreter +The same information is also available from the [`nix help-stores`](./nix3-help-stores.md) command. + +# Shebang interpreter The `nix` command can be used as a `#!` interpreter. Arguments to Nix can be passed on subsequent lines in the script. From 333ea684b065318aa49aec367c995b3d8c5d65ed Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Fri, 1 Dec 2023 01:39:52 +0100 Subject: [PATCH 030/141] Add boost::regex regression test --- src/libexpr/tests/primops.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/libexpr/tests/primops.cc b/src/libexpr/tests/primops.cc index d820b860e..7485fa0d0 100644 --- a/src/libexpr/tests/primops.cc +++ b/src/libexpr/tests/primops.cc @@ -814,6 +814,14 @@ namespace nix { ASSERT_THAT(*v.listElems()[0], IsStringEq("FOO")); } + TEST_F(PrimOpTest, match5) { + // The regex "\\{}" is valid and matches the string "{}". + // Caused a regression before when trying to switch from std::regex to boost::regex. + // See https://github.com/NixOS/nix/pull/7762#issuecomment-1834303659 + auto v = eval("builtins.match \"\\\\{}\" \"{}\""); + ASSERT_THAT(v, IsListOfSize(0)); + } + TEST_F(PrimOpTest, attrNames) { auto v = eval("builtins.attrNames { x = 1; y = 2; z = 3; a = 2; }"); ASSERT_THAT(v, IsListOfSize(4)); From d5e934fb73496a2509755be5945a8bcf1730d59d Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 1 Dec 2023 01:54:48 +0100 Subject: [PATCH 031/141] add redirect to new store page --- doc/manual/_redirects | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/_redirects b/doc/manual/_redirects index 2038671d7..62c693c97 100644 --- a/doc/manual/_redirects +++ b/doc/manual/_redirects @@ -31,9 +31,9 @@ /installation/installation /installation 301! /package-management/basic-package-mgmt /command-ref/nix-env 301! -/package-management/channels* /command-ref/nix-channel 301! +/package-management/channels /command-ref/nix-channel 301! /package-management/package-management /package-management 301! -/package-management/s3-substituter* /command-ref/new-cli/nix3-help-stores#s3-binary-cache-store 301! +/package-management/s3-substituter /store/types/s3-binary-cache-store 301! /protocols/protocols /protocols 301! From eff9b12bc296213c3ba824e90869bcafc4103e1c Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Fri, 1 Dec 2023 11:25:22 +0000 Subject: [PATCH 032/141] Further changes --- binary-tarball.nix | 6 ++- coverage.nix | 35 ++++++++++++++ flake.nix | 115 ++++++++++----------------------------------- package.nix | 88 +++++++++++++++++----------------- 4 files changed, 108 insertions(+), 136 deletions(-) create mode 100644 coverage.nix diff --git a/binary-tarball.nix b/binary-tarball.nix index 1fa185519..0053abbca 100644 --- a/binary-tarball.nix +++ b/binary-tarball.nix @@ -1,8 +1,8 @@ { runCommand -, version , system -, nix +, buildPackages , cacert +, nix }: let @@ -11,6 +11,8 @@ let rootPaths = [ nix cacert ]; }; + inherit (nix) version; + env = { meta.description = "Distribution-independent Nix bootstrap binaries for ${system}"; }; diff --git a/coverage.nix b/coverage.nix new file mode 100644 index 000000000..2390ef52d --- /dev/null +++ b/coverage.nix @@ -0,0 +1,35 @@ +{ lib +, releaseTools +, nix +, stdenv +}: + +let + inherit (nix) version; + +in + +releaseTools.coverageAnalysis { + name = "nix-coverage-${version}"; + + inherit (nix) + src + configureFlags + nativeBuildInputs + buildInputs + #checkInputs + ; + + enableParallelBuilding = true; + + dontInstall = false; + + doInstallCheck = true; + installCheckTarget = "installcheck"; # work around buggy detection in stdenv + + lcovFilter = [ "*/boost/*" "*-tab.*" ]; + + hardeningDisable = ["fortify"]; + + NIX_CFLAGS_COMPILE = "-DCOVERAGE=1"; +} diff --git a/flake.nix b/flake.nix index 544a07ba6..c0841a76d 100644 --- a/flake.nix +++ b/flake.nix @@ -479,60 +479,25 @@ dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); # Line coverage analysis. - coverage = - with nixpkgsFor.x86_64-linux.native; - with commonDeps { inherit pkgs; }; - - releaseTools.coverageAnalysis { - name = "nix-coverage-${version}"; - - src = nixSrc; - - configureFlags = testConfigureFlags; - - enableParallelBuilding = true; - - nativeBuildInputs = nativeBuildDeps; - buildInputs = buildDeps ++ propagatedDeps ++ awsDeps ++ checkDeps; - - dontInstall = false; - - doInstallCheck = true; - installCheckTarget = "installcheck"; # work around buggy detection in stdenv - - lcovFilter = [ "*/boost/*" "*-tab.*" ]; - - hardeningDisable = ["fortify"]; - - NIX_CFLAGS_COMPILE = "-DCOVERAGE=1"; - }; + coverage = nixpkgsFor.x86_64-linux.native.callPackage ./coverage.nix {}; # API docs for Nix's unstable internal C++ interfaces. - internal-api-docs = - with nixpkgsFor.x86_64-linux.native; - with commonDeps { inherit pkgs; }; + internal-api-docs = nixpkgsFor.x86_64-linux.native.nix.overrideAttrs (old: { + pname = "nix-internal-api-docs"; - stdenv.mkDerivation { - pname = "nix-internal-api-docs"; - inherit version; + configureFlags = old.configureFlags ++ [ "--enable-internal-api-docs" ]; + nativeBuildInputs = old.nativeBuildInputs ++ [ nixpkgsFor.x86_64-linux.native.doxygen ]; - src = nixSrc; + dontBuild = true; + doCheck = false; - configureFlags = testConfigureFlags ++ internalApiDocsConfigureFlags; + installTargets = [ "internal-api-html" ]; - nativeBuildInputs = nativeBuildDeps; - buildInputs = buildDeps ++ propagatedDeps - ++ awsDeps ++ checkDeps ++ internalApiDocsDeps; - - dontBuild = true; - - installTargets = [ "internal-api-html" ]; - - postInstall = '' - mkdir -p $out/nix-support - echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> $out/nix-support/hydra-build-products - ''; - }; + postInstall = '' + mkdir -p $out/nix-support + echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> $out/nix-support/hydra-build-products + ''; + }); # System tests. tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // { @@ -540,7 +505,9 @@ # Make sure that nix-env still produces the exact same result # on a particular version of Nixpkgs. evalNixpkgs = - with nixpkgsFor.x86_64-linux.native; + let + inherit (nixpkgsFor.x86_64-linux.native) runCommand nix nixpkgs-regression; + in runCommand "eval-nixos" { buildInputs = [ nix ]; } '' type -p nix-env @@ -627,47 +594,17 @@ stdenvs))); devShells = let - makeShell = pkgs: stdenv: - let - canRunInstalled = stdenv.buildPlatform.canExecute stdenv.hostPlatform; - in - with commonDeps { inherit pkgs; }; - stdenv.mkDerivation { - name = "nix"; + makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; }).overrideAttrs (_: { + installFlags = "sysconfdir=$(out)/etc"; + shellHook = '' + PATH=$prefix/bin:$PATH + unset PYTHONPATH + export MANPATH=$out/share/man:$MANPATH - outputs = [ "out" "dev" "doc" ] - ++ lib.optional (stdenv.hostPlatform != stdenv.buildPlatform) "check"; - - nativeBuildInputs = nativeBuildDeps - ++ lib.optional stdenv.cc.isClang pkgs.buildPackages.bear - ++ lib.optional - (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) - pkgs.buildPackages.clang-tools - # We want changelog-d in the shell even if the current build doesn't need it - ++ lib.optional (officialRelease || ! buildUnreleasedNotes) changelog-d - ; - - buildInputs = buildDeps ++ propagatedDeps - ++ awsDeps ++ checkDeps ++ internalApiDocsDeps; - - configureFlags = configureFlags - ++ testConfigureFlags ++ internalApiDocsConfigureFlags - ++ lib.optional (!canRunInstalled) "--disable-doc-gen"; - - enableParallelBuilding = true; - - installFlags = "sysconfdir=$(out)/etc"; - - shellHook = - '' - PATH=$prefix/bin:$PATH - unset PYTHONPATH - export MANPATH=$out/share/man:$MANPATH - - # Make bash completion work. - XDG_DATA_DIRS+=:$out/share - ''; - }; + # Make bash completion work. + XDG_DATA_DIRS+=:$out/share + ''; + }); in forAllSystems (system: let diff --git a/package.nix b/package.nix index 8d62120fb..bed77ba3b 100644 --- a/package.nix +++ b/package.nix @@ -41,16 +41,12 @@ }: let - version = lib.fileContents ./.version + versionSuffix; - - inherit (stdenv.hostPlatform) isStatic; - canRunInstalled = stdenv.buildPlatform.canExecute stdenv.hostPlatform; in stdenv.mkDerivation (finalAttrs: { - name = "nix-${version}"; + pname = "nix"; inherit version; src = @@ -103,17 +99,12 @@ stdenv.mkDerivation (finalAttrs: { bison flex (lib.getBin lowdown) + jq # Also for custom mdBook preprocessor. mdbook mdbook-linkcheck autoconf-archive autoreconfHook pkg-config - - # Tests - git - mercurial # FIXME: remove? only needed for tests - jq # Also for custom mdBook preprocessor. - openssh # only needed for tests (ssh-keygen) ] ++ lib.optional stdenv.hostPlatform.isLinux util-linux # Official releases don't have rl-next, so we don't need to compile a changelog @@ -133,19 +124,29 @@ stdenv.mkDerivation (finalAttrs: { sqlite xz ] - ++ lib.optionals stdenv.isLinux [libseccomp] + ++ lib.optional stdenv.isLinux libseccomp ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid # There have been issues building these dependencies - ++ lib.optionals (stdenv.hostPlatform == stdenv.buildPlatform) (lib.optional (stdenv.isLinux || stdenv.isDarwin) + ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin)) (aws-sdk-cpp.override { apis = ["s3" "transfer"]; customMemoryManagement = false; - })) - ++ lib.optionals finalAttrs.doCheck [ + }) + ; + + doCheck = true; + + checkInputs = [ gtest rapidcheck ]; + nativeCheckInputs = [ + git + mercurial # FIXME: remove? only needed for tests + openssh # only needed for tests (ssh-keygen) + ]; + propagatedBuildInputs = [ boehmgc nlohmann_json @@ -153,52 +154,49 @@ stdenv.mkDerivation (finalAttrs: { disallowedReferences = [ boost ]; - preConfigure = lib.optionalString (! stdenv.hostPlatform.isStatic) - '' - # Copy libboost_context so we don't get all of Boost in our closure. - # https://github.com/NixOS/nixpkgs/issues/45462 - mkdir -p $out/lib - cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*,libboost_regex*} $out/lib - rm -f $out/lib/*.a - ${lib.optionalString stdenv.hostPlatform.isLinux '' - chmod u+w $out/lib/*.so.* - patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* - ''} - ${lib.optionalString stdenv.hostPlatform.isDarwin '' - for LIB in $out/lib/*.dylib; do - chmod u+w $LIB - install_name_tool -id $LIB $LIB - install_name_tool -delete_rpath ${boost}/lib/ $LIB || true - done - install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib - ''} - ''; + preConfigure = lib.optionalString (! stdenv.hostPlatform.isStatic) '' + # Copy libboost_context so we don't get all of Boost in our closure. + # https://github.com/NixOS/nixpkgs/issues/45462 + mkdir -p $out/lib + cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*,libboost_regex*} $out/lib + rm -f $out/lib/*.a + ${lib.optionalString stdenv.hostPlatform.isLinux '' + chmod u+w $out/lib/*.so.* + patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* + ''} + ${lib.optionalString stdenv.hostPlatform.isDarwin '' + for LIB in $out/lib/*.dylib; do + chmod u+w $LIB + install_name_tool -id $LIB $LIB + install_name_tool -delete_rpath ${boost}/lib/ $LIB || true + done + install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib + ''} + ''; configureFlags = lib.optionals stdenv.isLinux [ "--with-boost=${boost}/lib" "--with-sandbox-shell=${sh}/bin/busybox" ] - ++ lib.optionals (stdenv.isLinux && !(isStatic && stdenv.system == "aarch64-linux")) [ + ++ lib.optional (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) "LDFLAGS=-fuse-ld=gold" - ] ++ [ "--sysconfdir=/etc" ] ++ lib.optional stdenv.hostPlatform.isStatic "--enable-embedded-sandbox-shell" ++ [ (lib.enableFeature finalAttrs.doCheck "tests") ] - ++ lib.optionals finalAttrs.doCheck ([ "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include" ] - ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ - "--enable-install-unit-tests" - "--with-check-bin-dir=${builtins.placeholder "check"}/bin" - "--with-check-lib-dir=${builtins.placeholder "check"}/lib" - ]) + ++ lib.optionals finalAttrs.doCheck ( + [ "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include" ] + ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ + "--enable-install-unit-tests" + "--with-check-bin-dir=${builtins.placeholder "check"}/bin" + "--with-check-lib-dir=${builtins.placeholder "check"}/lib" + ]) ++ lib.optional (!canRunInstalled) "--disable-doc-gen"; enableParallelBuilding = true; makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1"; - doCheck = true; - installFlags = "sysconfdir=$(out)/etc"; postInstall = '' From ea2dd166235e049699cf7f70c243c2b83089f824 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 1 Dec 2023 15:35:21 +0100 Subject: [PATCH 033/141] Use a proper enum rather than a boolean in runProgramInStore Makes the call-site much easier to understand. --- src/nix/develop.cc | 2 +- src/nix/fmt.cc | 2 +- src/nix/run.cc | 8 ++++---- src/nix/run.hh | 7 ++++++- 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/nix/develop.cc b/src/nix/develop.cc index ae9be79a3..606b044b0 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -662,7 +662,7 @@ struct CmdDevelop : Common, MixEnvironment } } - runProgramInStore(store, true, shell, args, buildEnvironment.getSystem()); + runProgramInStore(store, UseSearchPath::Use, shell, args, buildEnvironment.getSystem()); } }; diff --git a/src/nix/fmt.cc b/src/nix/fmt.cc index 396c93dbb..059904150 100644 --- a/src/nix/fmt.cc +++ b/src/nix/fmt.cc @@ -49,7 +49,7 @@ struct CmdFmt : SourceExprCommand { } } - runProgramInStore(store, false, app.program, programArgs); + runProgramInStore(store, UseSearchPath::DontUse, app.program, programArgs); }; }; diff --git a/src/nix/run.cc b/src/nix/run.cc index d531f712d..efc0c56a1 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -25,7 +25,7 @@ std::string chrootHelperName = "__run_in_chroot"; namespace nix { void runProgramInStore(ref store, - bool search, + UseSearchPath useSearchPath, const std::string & program, const Strings & args, std::optional system) @@ -59,7 +59,7 @@ void runProgramInStore(ref store, if (system) setPersonality(*system); - if (search) + if (useSearchPath == UseSearchPath::Use) execvp(program.c_str(), stringsToCharPtrs(args).data()); else execv(program.c_str(), stringsToCharPtrs(args).data()); @@ -136,7 +136,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment Strings args; for (auto & arg : command) args.push_back(arg); - runProgramInStore(store, true, *command.begin(), args); + runProgramInStore(store, UseSearchPath::Use, *command.begin(), args); } }; @@ -198,7 +198,7 @@ struct CmdRun : InstallableValueCommand Strings allArgs{app.program}; for (auto & i : args) allArgs.push_back(i); - runProgramInStore(store, false, app.program, allArgs); + runProgramInStore(store, UseSearchPath::DontUse, app.program, allArgs); } }; diff --git a/src/nix/run.hh b/src/nix/run.hh index c62287e7e..a55917b06 100644 --- a/src/nix/run.hh +++ b/src/nix/run.hh @@ -5,8 +5,13 @@ namespace nix { +enum struct UseSearchPath { + Use, + DontUse +}; + void runProgramInStore(ref store, - bool search, + UseSearchPath useSearchPath, const std::string & program, const Strings & args, std::optional system = std::nullopt); From d59bdbe4fd757d99b6625db1d3560a39a371d9e9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 1 Dec 2023 10:20:19 -0500 Subject: [PATCH 034/141] Add two missing `#include "nar-info.hh"` GitHub's racy CI caused this oversight to sneak through. --- src/libstore/tests/nar-info.cc | 1 + src/nix/path-info.cc | 1 + 2 files changed, 2 insertions(+) diff --git a/src/libstore/tests/nar-info.cc b/src/libstore/tests/nar-info.cc index c5b21d56b..88e6e1add 100644 --- a/src/libstore/tests/nar-info.cc +++ b/src/libstore/tests/nar-info.cc @@ -2,6 +2,7 @@ #include #include "path-info.hh" +#include "nar-info.hh" #include "tests/characterization.hh" #include "tests/libstore.hh" diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 080d6bbf1..5f10cfb61 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -2,6 +2,7 @@ #include "shared.hh" #include "store-api.hh" #include "common-args.hh" +#include "nar-info.hh" #include #include From 91b6833686a6a6d9eac7f3f66393ec89ef1d3b57 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 25 Aug 2023 10:20:28 -0400 Subject: [PATCH 035/141] Move tests to separate directories, and document Today, with the tests inside a `tests` intermingled with the corresponding library's source code, we have a few problems: - We have to be careful that wildcards don't end up with tests being built as part of Nix proper, or test headers being installed as part of Nix proper. - Tests in libraries but not executables is not right: - It means each executable runs the previous unit tests again, because it needs the libraries. - It doesn't work right on Windows, which doesn't want you to load a DLL just for the side global variable . It could be made to work with the dlopen equivalent, but that's gross! This reorg solves these problems. There is a remaining problem which is that sibbling headers (like `hash.hh` the test header vs `hash.hh` the main `libnixutil` header) end up shadowing each other. This PR doesn't solve that. That is left as future work for a future PR. Co-authored-by: Valentin Gagarin --- .gitignore | 6 +- Makefile | 10 ++- doc/internal-api/doxygen.cfg.in | 12 ++- doc/manual/src/contributing/testing.md | 67 ++++++++++---- flake.nix | 2 +- mk/common-test.sh | 2 +- mk/programs.mk | 2 +- src/libexpr/tests/local.mk | 23 ----- src/libstore/tests/local.mk | 37 -------- src/libutil/tests/local.mk | 41 --------- tests/unit/libexpr-support/local.mk | 23 +++++ .../unit/libexpr-support}/tests/libexpr.hh | 0 .../libexpr-support/tests/value/context.cc | 30 +++++++ .../libexpr-support}/tests/value/context.hh | 2 +- .../unit/libexpr}/derived-path.cc | 0 .../unit/libexpr}/error_traces.cc | 0 .../tests => tests/unit/libexpr}/flakeref.cc | 0 .../tests => tests/unit/libexpr}/json.cc | 0 tests/unit/libexpr/local.mk | 36 ++++++++ .../tests => tests/unit/libexpr}/primops.cc | 0 .../unit/libexpr}/search-path.cc | 0 .../tests => tests/unit/libexpr}/trivial.cc | 0 .../unit/libexpr}/value/context.cc | 30 ------- .../unit/libexpr}/value/print.cc | 0 tests/unit/libstore-support/local.mk | 21 +++++ .../libstore-support/tests/derived-path.cc | 57 ++++++++++++ .../libstore-support}/tests/derived-path.hh | 0 .../unit/libstore-support}/tests/libstore.hh | 0 .../libstore-support/tests/outputs-spec.cc | 24 +++++ .../libstore-support}/tests/outputs-spec.hh | 2 +- tests/unit/libstore-support/tests/path.cc | 82 ++++++++++++++++++ .../unit/libstore-support}/tests/path.hh | 3 + .../unit/libstore-support}/tests/protocol.hh | 2 +- .../unit/libstore}/common-protocol.cc | 0 .../data}/common-protocol/content-address.bin | Bin .../data}/common-protocol/drv-output.bin | Bin .../optional-content-address.bin | Bin .../common-protocol/optional-store-path.bin | Bin .../data}/common-protocol/realisation.bin | Bin .../libstore/data}/common-protocol/set.bin | Bin .../data}/common-protocol/store-path.bin | Bin .../libstore/data}/common-protocol/string.bin | Bin .../libstore/data}/common-protocol/vector.bin | Bin .../derivation/bad-old-version-dyn-deps.drv | 0 .../libstore/data}/derivation/bad-version.drv | 0 .../data}/derivation/dynDerivationDeps.drv | 0 .../data}/derivation/dynDerivationDeps.json | 0 .../data}/derivation/output-caFixedFlat.json | 0 .../data}/derivation/output-caFixedNAR.json | 0 .../data}/derivation/output-caFixedText.json | 0 .../data}/derivation/output-caFloating.json | 0 .../data}/derivation/output-deferred.json | 0 .../data}/derivation/output-impure.json | 0 .../derivation/output-inputAddressed.json | 0 .../unit/libstore/data}/derivation/simple.drv | 0 .../libstore/data}/derivation/simple.json | 0 .../unit/libstore/data}/nar-info/impure.json | 0 .../unit/libstore/data}/nar-info/pure.json | 0 .../unit/libstore/data}/path-info/impure.json | 0 .../unit/libstore/data}/path-info/pure.json | 0 .../data}/serve-protocol/build-result-2.2.bin | Bin .../data}/serve-protocol/build-result-2.3.bin | Bin .../data}/serve-protocol/build-result-2.6.bin | Bin .../data}/serve-protocol/content-address.bin | Bin .../data}/serve-protocol/drv-output.bin | Bin .../optional-content-address.bin | Bin .../serve-protocol/optional-store-path.bin | Bin .../data}/serve-protocol/realisation.bin | Bin .../libstore/data}/serve-protocol/set.bin | Bin .../data}/serve-protocol/store-path.bin | Bin .../libstore/data}/serve-protocol/string.bin | Bin .../libstore/data}/serve-protocol/vector.bin | Bin .../worker-protocol/build-result-1.27.bin | Bin .../worker-protocol/build-result-1.28.bin | Bin .../worker-protocol/build-result-1.29.bin | Bin .../data}/worker-protocol/content-address.bin | Bin .../worker-protocol/derived-path-1.29.bin | Bin .../worker-protocol/derived-path-1.30.bin | Bin .../data}/worker-protocol/drv-output.bin | Bin .../keyed-build-result-1.29.bin | Bin .../optional-content-address.bin | Bin .../worker-protocol/optional-store-path.bin | Bin .../worker-protocol/optional-trusted-flag.bin | Bin .../data}/worker-protocol/realisation.bin | Bin .../libstore/data}/worker-protocol/set.bin | Bin .../data}/worker-protocol/store-path.bin | Bin .../libstore/data}/worker-protocol/string.bin | Bin .../unkeyed-valid-path-info-1.15.bin | Bin .../worker-protocol/valid-path-info-1.15.bin | Bin .../worker-protocol/valid-path-info-1.16.bin | Bin .../libstore/data}/worker-protocol/vector.bin | Bin .../unit/libstore}/derivation.cc | 2 +- .../unit/libstore}/derived-path.cc | 53 ----------- .../unit/libstore}/downstream-placeholder.cc | 0 tests/unit/libstore/local.mk | 31 +++++++ .../tests => tests/unit/libstore}/machines.cc | 4 +- .../unit/libstore}/nar-info-disk-cache.cc | 0 .../tests => tests/unit/libstore}/nar-info.cc | 2 +- .../unit/libstore}/outputs-spec.cc | 27 +----- .../unit/libstore}/path-info.cc | 2 +- .../tests => tests/unit/libstore}/path.cc | 73 ---------------- .../unit/libstore}/references.cc | 0 .../unit/libstore}/serve-protocol.cc | 0 .../libstore}/test-data/machines.bad_format | 0 .../unit/libstore}/test-data/machines.valid | 0 .../unit/libstore}/worker-protocol.cc | 0 tests/unit/libutil-support/local.mk | 19 ++++ .../tests/characterization.hh | 4 +- tests/unit/libutil-support/tests/hash.cc | 20 +++++ .../unit/libutil-support}/tests/hash.hh | 0 .../tests => tests/unit/libutil}/args.cc | 6 +- .../unit/libutil}/canon-path.cc | 0 .../unit/libutil}/chunked-vector.cc | 0 .../tests => tests/unit/libutil}/closure.cc | 0 .../unit/libutil}/compression.cc | 0 .../tests => tests/unit/libutil}/config.cc | 0 .../unit/libutil/data}/git/check-data.sh | 2 +- .../libutil/data}/git/hello-world-blob.bin | Bin .../unit/libutil/data}/git/hello-world.bin | Bin .../unit/libutil/data}/git/tree.bin | Bin .../unit/libutil/data}/git/tree.txt | 0 .../tests => tests/unit/libutil}/git.cc | 6 +- .../tests => tests/unit/libutil}/hash.cc | 20 +---- .../tests => tests/unit/libutil}/hilite.cc | 0 .../unit/libutil}/json-utils.cc | 0 tests/unit/libutil/local.mk | 31 +++++++ .../tests => tests/unit/libutil}/logging.cc | 0 .../tests => tests/unit/libutil}/lru-cache.cc | 0 .../tests => tests/unit/libutil}/pool.cc | 0 .../unit/libutil}/references.cc | 0 .../unit/libutil}/suggestions.cc | 0 .../tests => tests/unit/libutil}/tests.cc | 0 .../tests => tests/unit/libutil}/url.cc | 0 .../unit/libutil}/xml-writer.cc | 0 134 files changed, 464 insertions(+), 352 deletions(-) delete mode 100644 src/libexpr/tests/local.mk delete mode 100644 src/libstore/tests/local.mk delete mode 100644 src/libutil/tests/local.mk create mode 100644 tests/unit/libexpr-support/local.mk rename {src/libexpr => tests/unit/libexpr-support}/tests/libexpr.hh (100%) create mode 100644 tests/unit/libexpr-support/tests/value/context.cc rename {src/libexpr => tests/unit/libexpr-support}/tests/value/context.hh (95%) rename {src/libexpr/tests => tests/unit/libexpr}/derived-path.cc (100%) rename {src/libexpr/tests => tests/unit/libexpr}/error_traces.cc (100%) rename {src/libexpr/tests => tests/unit/libexpr}/flakeref.cc (100%) rename {src/libexpr/tests => tests/unit/libexpr}/json.cc (100%) create mode 100644 tests/unit/libexpr/local.mk rename {src/libexpr/tests => tests/unit/libexpr}/primops.cc (100%) rename {src/libexpr/tests => tests/unit/libexpr}/search-path.cc (100%) rename {src/libexpr/tests => tests/unit/libexpr}/trivial.cc (100%) rename {src/libexpr/tests => tests/unit/libexpr}/value/context.cc (83%) rename {src/libexpr/tests => tests/unit/libexpr}/value/print.cc (100%) create mode 100644 tests/unit/libstore-support/local.mk create mode 100644 tests/unit/libstore-support/tests/derived-path.cc rename {src/libstore => tests/unit/libstore-support}/tests/derived-path.hh (100%) rename {src/libstore => tests/unit/libstore-support}/tests/libstore.hh (100%) create mode 100644 tests/unit/libstore-support/tests/outputs-spec.cc rename {src/libstore => tests/unit/libstore-support}/tests/outputs-spec.hh (89%) create mode 100644 tests/unit/libstore-support/tests/path.cc rename {src/libstore => tests/unit/libstore-support}/tests/path.hh (82%) rename {src/libstore => tests/unit/libstore-support}/tests/protocol.hh (96%) rename {src/libstore/tests => tests/unit/libstore}/common-protocol.cc (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/content-address.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/drv-output.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/optional-content-address.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/optional-store-path.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/realisation.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/set.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/store-path.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/string.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/common-protocol/vector.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/bad-old-version-dyn-deps.drv (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/bad-version.drv (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/dynDerivationDeps.drv (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/dynDerivationDeps.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-caFixedFlat.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-caFixedNAR.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-caFixedText.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-caFloating.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-deferred.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-impure.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/output-inputAddressed.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/simple.drv (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/derivation/simple.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/nar-info/impure.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/nar-info/pure.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/path-info/impure.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/path-info/pure.json (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/build-result-2.2.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/build-result-2.3.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/build-result-2.6.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/content-address.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/drv-output.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/optional-content-address.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/optional-store-path.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/realisation.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/set.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/store-path.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/string.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/serve-protocol/vector.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/build-result-1.27.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/build-result-1.28.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/build-result-1.29.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/content-address.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/derived-path-1.29.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/derived-path-1.30.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/drv-output.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/keyed-build-result-1.29.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/optional-content-address.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/optional-store-path.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/optional-trusted-flag.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/realisation.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/set.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/store-path.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/string.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/unkeyed-valid-path-info-1.15.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/valid-path-info-1.15.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/valid-path-info-1.16.bin (100%) rename {unit-test-data/libstore => tests/unit/libstore/data}/worker-protocol/vector.bin (100%) rename {src/libstore/tests => tests/unit/libstore}/derivation.cc (99%) rename {src/libstore/tests => tests/unit/libstore}/derived-path.cc (66%) rename {src/libstore/tests => tests/unit/libstore}/downstream-placeholder.cc (100%) create mode 100644 tests/unit/libstore/local.mk rename {src/libstore/tests => tests/unit/libstore}/machines.cc (97%) rename {src/libstore/tests => tests/unit/libstore}/nar-info-disk-cache.cc (100%) rename {src/libstore/tests => tests/unit/libstore}/nar-info.cc (98%) rename {src/libstore/tests => tests/unit/libstore}/outputs-spec.cc (92%) rename {src/libstore/tests => tests/unit/libstore}/path-info.cc (97%) rename {src/libstore/tests => tests/unit/libstore}/path.cc (59%) rename {src/libstore/tests => tests/unit/libstore}/references.cc (100%) rename {src/libstore/tests => tests/unit/libstore}/serve-protocol.cc (100%) rename {src/libstore/tests => tests/unit/libstore}/test-data/machines.bad_format (100%) rename {src/libstore/tests => tests/unit/libstore}/test-data/machines.valid (100%) rename {src/libstore/tests => tests/unit/libstore}/worker-protocol.cc (100%) create mode 100644 tests/unit/libutil-support/local.mk rename {src/libutil => tests/unit/libutil-support}/tests/characterization.hh (95%) create mode 100644 tests/unit/libutil-support/tests/hash.cc rename {src/libutil => tests/unit/libutil-support}/tests/hash.hh (100%) rename {src/libutil/tests => tests/unit/libutil}/args.cc (98%) rename {src/libutil/tests => tests/unit/libutil}/canon-path.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/chunked-vector.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/closure.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/compression.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/config.cc (100%) rename {unit-test-data/libutil => tests/unit/libutil/data}/git/check-data.sh (98%) rename {unit-test-data/libutil => tests/unit/libutil/data}/git/hello-world-blob.bin (100%) rename {unit-test-data/libutil => tests/unit/libutil/data}/git/hello-world.bin (100%) rename {unit-test-data/libutil => tests/unit/libutil/data}/git/tree.bin (100%) rename {unit-test-data/libutil => tests/unit/libutil/data}/git/tree.txt (100%) rename {src/libutil/tests => tests/unit/libutil}/git.cc (97%) rename {src/libutil/tests => tests/unit/libutil}/hash.cc (92%) rename {src/libutil/tests => tests/unit/libutil}/hilite.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/json-utils.cc (100%) create mode 100644 tests/unit/libutil/local.mk rename {src/libutil/tests => tests/unit/libutil}/logging.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/lru-cache.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/pool.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/references.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/suggestions.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/tests.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/url.cc (100%) rename {src/libutil/tests => tests/unit/libutil}/xml-writer.cc (100%) diff --git a/.gitignore b/.gitignore index 767a5d6ed..d9f9d949b 100644 --- a/.gitignore +++ b/.gitignore @@ -45,14 +45,14 @@ perl/Makefile.config /src/libexpr/parser-tab.hh /src/libexpr/parser-tab.output /src/libexpr/nix.tbl -/src/libexpr/tests/libnixexpr-tests +/tests/unit/libexpr/libnixexpr-tests # /src/libstore/ *.gen.* -/src/libstore/tests/libnixstore-tests +/tests/unit/libstore/libnixstore-tests # /src/libutil/ -/src/libutil/tests/libnixutil-tests +/tests/unit/libutil/libnixutil-tests /src/nix/nix diff --git a/Makefile b/Makefile index 92727bea5..eea297c89 100644 --- a/Makefile +++ b/Makefile @@ -25,11 +25,13 @@ makefiles = \ endif ifeq ($(ENABLE_BUILD)_$(ENABLE_TESTS), yes_yes) -UNIT_TEST_ENV = _NIX_TEST_UNIT_DATA=unit-test-data makefiles += \ - src/libutil/tests/local.mk \ - src/libstore/tests/local.mk \ - src/libexpr/tests/local.mk + tests/unit/libutil/local.mk \ + tests/unit/libutil-support/local.mk \ + tests/unit/libstore/local.mk \ + tests/unit/libstore-support/local.mk \ + tests/unit/libexpr/local.mk \ + tests/unit/libexpr-support/local.mk endif ifeq ($(ENABLE_TESTS), yes) diff --git a/doc/internal-api/doxygen.cfg.in b/doc/internal-api/doxygen.cfg.in index 599be2470..ad5af97e6 100644 --- a/doc/internal-api/doxygen.cfg.in +++ b/doc/internal-api/doxygen.cfg.in @@ -39,17 +39,21 @@ INPUT = \ src/libcmd \ src/libexpr \ src/libexpr/flake \ - src/libexpr/tests \ - src/libexpr/tests/value \ + tests/unit/libexpr \ + tests/unit/libexpr/value \ + tests/unit/libexpr/test \ + tests/unit/libexpr/test/value \ src/libexpr/value \ src/libfetchers \ src/libmain \ src/libstore \ src/libstore/build \ src/libstore/builtins \ - src/libstore/tests \ + tests/unit/libstore \ + tests/unit/libstore/test \ src/libutil \ - src/libutil/tests \ + tests/unit/libutil \ + tests/unit/libutil/test \ src/nix \ src/nix-env \ src/nix-store diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index 0b45b88a3..d8d162379 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -20,6 +20,7 @@ The unit tests are defined using the [googletest] and [rapidcheck] frameworks. [googletest]: https://google.github.io/googletest/ [rapidcheck]: https://github.com/emil-e/rapidcheck +[property testing]: https://en.wikipedia.org/wiki/Property_testing ### Source and header layout @@ -28,34 +29,50 @@ The unit tests are defined using the [googletest] and [rapidcheck] frameworks. > ``` > src > ├── libexpr +> │ ├── local.mk > │ ├── value/context.hh > │ ├── value/context.cc +> │ … +> │ +> ├── tests > │ │ > │ … -> └── tests -> │ ├── value/context.hh -> │ ├── value/context.cc +> │ └── unit +> │ ├── libutil +> │ │ ├── local.mk +> │ │ … +> │ │ └── data +> │ │ ├── git/tree.txt +> │ │ … > │ │ -> │ … -> │ -> ├── unit-test-data -> │ ├── libstore -> │ │ ├── worker-protocol/content-address.bin -> │ │ … -> │ … +> │ ├── libexpr-support +> │ │ ├── local.mk +> │ │ └── tests +> │ │ ├── value/context.hh +> │ │ ├── value/context.cc +> │ │ … +> │ │ +> │ ├── libexpr +> │ … ├── local.mk +> │ ├── value/context.cc +> │ … > … > ``` -The unit tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `src/${library_shortname}/tests` within the directory for the library (`src/${library_shortname}`). +The tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `tests/unit/${library_name_without-nix}`. +Given a interface (header) and implementation pair in the original library, say, `src/libexpr/value/context.{hh,cc}`, we write tests for it in `tests/unit/libexpr/tests/value/context.cc`, and (possibly) declare/define additional interfaces for testing purposes in `tests/unit/libexpr-support/tests/value/context.{hh,cc}`. -The data is in `unit-test-data`, with one subdir per library, with the same name as where the code goes. -For example, `libnixstore` code is in `src/libstore`, and its test data is in `unit-test-data/libstore`. -The path to the `unit-test-data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. +Data for unit tests is stored in a `data` subdir of the directory for each unit test executable. +For example, `libnixstore` code is in `src/libstore`, and its test data is in `tests/unit/libstore/data`. +The path to the `tests/unit/data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. +Note that each executable only gets the data for its tests. -> **Note** -> Due to the way googletest works, downstream unit test executables will actually include and re-run upstream library tests. -> Therefore it is important that the same value for `_NIX_TEST_UNIT_DATA` be used with the tests for each library. -> That is why we have the test data nested within a single `unit-test-data` directory. +The unit test libraries are in `tests/unit/${library_name_without-nix}-lib`. +All headers are in a `tests` subdirectory so they are included with `#include "tests/"`. + +The use of all these separate directories for the unit tests might seem inconvenient, as for example the tests are not "right next to" the part of the code they are testing. +But organizing the tests this way has one big benefit: +there is no risk of any build-system wildcards for the library accidentally picking up test code that should not built and installed as part of the library. ### Running tests @@ -69,7 +86,7 @@ See [functional characterisation testing](#characterisation-testing-functional) Like with the functional characterisation, `_NIX_TEST_ACCEPT=1` is also used. For example: ```shell-session -$ _NIX_TEST_ACCEPT=1 make libstore-tests-exe_RUN +$ _NIX_TEST_ACCEPT=1 make libstore-tests_RUN ... [ SKIPPED ] WorkerProtoTest.string_read [ SKIPPED ] WorkerProtoTest.string_write @@ -80,6 +97,18 @@ $ _NIX_TEST_ACCEPT=1 make libstore-tests-exe_RUN will regenerate the "golden master" expected result for the `libnixstore` characterisation tests. The characterisation tests will mark themselves "skipped" since they regenerated the expected result instead of actually testing anything. +### Unit test support libraries + +There are headers and code which are not just used to test the library in question, but also downstream libraries. +For example, we do [property testing] with the [rapidcheck] library. +This requires writing `Arbitrary` "instances", which are used to describe how to generate values of a given type for the sake of running property tests. +Because types contain other types, `Arbitrary` "instances" for some type are not just useful for testing that type, but also any other type that contains it. +Downstream types frequently contain upstream types, so it is very important that we share arbitrary instances so that downstream libraries' property tests can also use them. + +It is important that these testing libraries don't contain any actual tests themselves. +On some platforms they would be run as part of every test executable that uses them, which is redundant. +On other platforms they wouldn't be run at all. + ## Functional tests The functional tests reside under the `tests/functional` directory and are listed in `tests/functional/local.mk`. diff --git a/flake.nix b/flake.nix index 822b3d31e..e2e510cbc 100644 --- a/flake.nix +++ b/flake.nix @@ -93,7 +93,7 @@ ./misc ./precompiled-headers.h ./src - ./unit-test-data + ./tests/unit ./COPYING ./scripts/local.mk functionalTestFiles diff --git a/mk/common-test.sh b/mk/common-test.sh index 00ccd1584..2783d293b 100644 --- a/mk/common-test.sh +++ b/mk/common-test.sh @@ -1,7 +1,7 @@ # Remove overall test dir (at most one of the two should match) and # remove file extension. test_name=$(echo -n "$test" | sed \ - -e "s|^unit-test-data/||" \ + -e "s|^tests/unit/[^/]*/data/||" \ -e "s|^tests/functional/||" \ -e "s|\.sh$||" \ ) diff --git a/mk/programs.mk b/mk/programs.mk index a88d9d949..6235311e9 100644 --- a/mk/programs.mk +++ b/mk/programs.mk @@ -87,6 +87,6 @@ define build-program # Phony target to run this program (typically as a dependency of 'check'). .PHONY: $(1)_RUN $(1)_RUN: $$($(1)_PATH) - $(trace-test) $$(UNIT_TEST_ENV) $$($(1)_PATH) + $(trace-test) $$($(1)_ENV) $$($(1)_PATH) endef diff --git a/src/libexpr/tests/local.mk b/src/libexpr/tests/local.mk deleted file mode 100644 index 7689a03e0..000000000 --- a/src/libexpr/tests/local.mk +++ /dev/null @@ -1,23 +0,0 @@ -check: libexpr-tests_RUN - -programs += libexpr-tests - -libexpr-tests_NAME := libnixexpr-tests - -libexpr-tests_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libexpr-tests_INSTALL_DIR := $(checkbindir) -else - libexpr-tests_INSTALL_DIR := -endif - -libexpr-tests_SOURCES := \ - $(wildcard $(d)/*.cc) \ - $(wildcard $(d)/value/*.cc) - -libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests -I src/libfetchers - -libexpr-tests_LIBS = libstore-tests libutils-tests libexpr libutil libstore libfetchers - -libexpr-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock diff --git a/src/libstore/tests/local.mk b/src/libstore/tests/local.mk deleted file mode 100644 index e9b8b4f99..000000000 --- a/src/libstore/tests/local.mk +++ /dev/null @@ -1,37 +0,0 @@ -check: libstore-tests-exe_RUN - -programs += libstore-tests-exe - -libstore-tests-exe_NAME = libnixstore-tests - -libstore-tests-exe_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libstore-tests-exe_INSTALL_DIR := $(checkbindir) -else - libstore-tests-exe_INSTALL_DIR := -endif - -libstore-tests-exe_LIBS = libstore-tests - -libstore-tests-exe_LDFLAGS := $(GTEST_LIBS) - -libraries += libstore-tests - -libstore-tests_NAME = libnixstore-tests - -libstore-tests_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libstore-tests_INSTALL_DIR := $(checklibdir) -else - libstore-tests_INSTALL_DIR := -endif - -libstore-tests_SOURCES := $(wildcard $(d)/*.cc) - -libstore-tests_CXXFLAGS += -I src/libstore -I src/libutil - -libstore-tests_LIBS = libutil-tests libstore libutil - -libstore-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk deleted file mode 100644 index 66886c45f..000000000 --- a/src/libutil/tests/local.mk +++ /dev/null @@ -1,41 +0,0 @@ -check: libutil-tests-exe_RUN - -programs += libutil-tests-exe - -libutil-tests-exe_NAME = libnixutil-tests - -libutil-tests-exe_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libutil-tests-exe_INSTALL_DIR := $(checkbindir) -else - libutil-tests-exe_INSTALL_DIR := -endif - -libutil-tests-exe_LIBS = libutil-tests - -libutil-tests-exe_LDFLAGS := $(GTEST_LIBS) - -libraries += libutil-tests - -libutil-tests_NAME = libnixutil-tests - -libutil-tests_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libutil-tests_INSTALL_DIR := $(checklibdir) -else - libutil-tests_INSTALL_DIR := -endif - -libutil-tests_SOURCES := $(wildcard $(d)/*.cc) - -libutil-tests_CXXFLAGS += -I src/libutil - -libutil-tests_LIBS = libutil - -libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) - -check: unit-test-data/libutil/git/check-data.sh.test - -$(eval $(call run-test,unit-test-data/libutil/git/check-data.sh)) diff --git a/tests/unit/libexpr-support/local.mk b/tests/unit/libexpr-support/local.mk new file mode 100644 index 000000000..28e87b8f2 --- /dev/null +++ b/tests/unit/libexpr-support/local.mk @@ -0,0 +1,23 @@ +libraries += libexpr-test-support + +libexpr-test-support_NAME = libnixexpr-test-support + +libexpr-test-support_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libexpr-test-support_INSTALL_DIR := $(checklibdir) +else + libexpr-test-support_INSTALL_DIR := +endif + +libexpr-test-support_SOURCES := \ + $(wildcard $(d)/tests/*.cc) \ + $(wildcard $(d)/tests/value/*.cc) + +libexpr-test-support_CXXFLAGS += $(libexpr-tests_EXTRA_INCLUDES) + +libexpr-test-support_LIBS = \ + libstore-test-support libutil-test-support \ + libexpr libstore libutil + +libexpr-test-support_LDFLAGS := -lrapidcheck diff --git a/src/libexpr/tests/libexpr.hh b/tests/unit/libexpr-support/tests/libexpr.hh similarity index 100% rename from src/libexpr/tests/libexpr.hh rename to tests/unit/libexpr-support/tests/libexpr.hh diff --git a/tests/unit/libexpr-support/tests/value/context.cc b/tests/unit/libexpr-support/tests/value/context.cc new file mode 100644 index 000000000..8658bdaef --- /dev/null +++ b/tests/unit/libexpr-support/tests/value/context.cc @@ -0,0 +1,30 @@ +#include + +#include "tests/path.hh" +#include "tests/value/context.hh" + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + return gen::just(NixStringContextElem::DrvDeep { + .drvPath = *gen::arbitrary(), + }); +} + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, std::variant_size_v)) { + case 0: + return gen::just(*gen::arbitrary()); + case 1: + return gen::just(*gen::arbitrary()); + case 2: + return gen::just(*gen::arbitrary()); + default: + assert(false); + } +} + +} diff --git a/src/libexpr/tests/value/context.hh b/tests/unit/libexpr-support/tests/value/context.hh similarity index 95% rename from src/libexpr/tests/value/context.hh rename to tests/unit/libexpr-support/tests/value/context.hh index c0bc97ba3..8c68c78bb 100644 --- a/src/libexpr/tests/value/context.hh +++ b/tests/unit/libexpr-support/tests/value/context.hh @@ -3,7 +3,7 @@ #include -#include +#include "value/context.hh" namespace rc { using namespace nix; diff --git a/src/libexpr/tests/derived-path.cc b/tests/unit/libexpr/derived-path.cc similarity index 100% rename from src/libexpr/tests/derived-path.cc rename to tests/unit/libexpr/derived-path.cc diff --git a/src/libexpr/tests/error_traces.cc b/tests/unit/libexpr/error_traces.cc similarity index 100% rename from src/libexpr/tests/error_traces.cc rename to tests/unit/libexpr/error_traces.cc diff --git a/src/libexpr/tests/flakeref.cc b/tests/unit/libexpr/flakeref.cc similarity index 100% rename from src/libexpr/tests/flakeref.cc rename to tests/unit/libexpr/flakeref.cc diff --git a/src/libexpr/tests/json.cc b/tests/unit/libexpr/json.cc similarity index 100% rename from src/libexpr/tests/json.cc rename to tests/unit/libexpr/json.cc diff --git a/tests/unit/libexpr/local.mk b/tests/unit/libexpr/local.mk new file mode 100644 index 000000000..5743880d7 --- /dev/null +++ b/tests/unit/libexpr/local.mk @@ -0,0 +1,36 @@ +check: libexpr-tests_RUN + +programs += libexpr-tests + +libexpr-tests_NAME := libnixexpr-tests + +libexpr-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data + +libexpr-tests_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libexpr-tests_INSTALL_DIR := $(checkbindir) +else + libexpr-tests_INSTALL_DIR := +endif + +libexpr-tests_SOURCES := \ + $(wildcard $(d)/*.cc) \ + $(wildcard $(d)/value/*.cc) + +libexpr-tests_EXTRA_INCLUDES = \ + -I tests/unit/libexpr-support \ + -I tests/unit/libstore-support \ + -I tests/unit/libutil-support \ + -I src/libexpr \ + -I src/libfetchers \ + -I src/libstore \ + -I src/libutil + +libexpr-tests_CXXFLAGS += $(libexpr-tests_EXTRA_INCLUDES) + +libexpr-tests_LIBS = \ + libexpr-test-support libstore-test-support libutils-test-support \ + libexpr libfetchers libstore libutil + +libexpr-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock diff --git a/src/libexpr/tests/primops.cc b/tests/unit/libexpr/primops.cc similarity index 100% rename from src/libexpr/tests/primops.cc rename to tests/unit/libexpr/primops.cc diff --git a/src/libexpr/tests/search-path.cc b/tests/unit/libexpr/search-path.cc similarity index 100% rename from src/libexpr/tests/search-path.cc rename to tests/unit/libexpr/search-path.cc diff --git a/src/libexpr/tests/trivial.cc b/tests/unit/libexpr/trivial.cc similarity index 100% rename from src/libexpr/tests/trivial.cc rename to tests/unit/libexpr/trivial.cc diff --git a/src/libexpr/tests/value/context.cc b/tests/unit/libexpr/value/context.cc similarity index 83% rename from src/libexpr/tests/value/context.cc rename to tests/unit/libexpr/value/context.cc index 92d4889ab..761286dbd 100644 --- a/src/libexpr/tests/value/context.cc +++ b/tests/unit/libexpr/value/context.cc @@ -117,36 +117,6 @@ TEST(NixStringContextElemTest, built_built_xp) { NixStringContextElem::parse("!foo!bar!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"), MissingExperimentalFeature); } -} - -namespace rc { -using namespace nix; - -Gen Arbitrary::arbitrary() -{ - return gen::just(NixStringContextElem::DrvDeep { - .drvPath = *gen::arbitrary(), - }); -} - -Gen Arbitrary::arbitrary() -{ - switch (*gen::inRange(0, std::variant_size_v)) { - case 0: - return gen::just(*gen::arbitrary()); - case 1: - return gen::just(*gen::arbitrary()); - case 2: - return gen::just(*gen::arbitrary()); - default: - assert(false); - } -} - -} - -namespace nix { - #ifndef COVERAGE RC_GTEST_PROP( diff --git a/src/libexpr/tests/value/print.cc b/tests/unit/libexpr/value/print.cc similarity index 100% rename from src/libexpr/tests/value/print.cc rename to tests/unit/libexpr/value/print.cc diff --git a/tests/unit/libstore-support/local.mk b/tests/unit/libstore-support/local.mk new file mode 100644 index 000000000..d5d657c91 --- /dev/null +++ b/tests/unit/libstore-support/local.mk @@ -0,0 +1,21 @@ +libraries += libstore-test-support + +libstore-test-support_NAME = libnixstore-test-support + +libstore-test-support_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libstore-test-support_INSTALL_DIR := $(checklibdir) +else + libstore-test-support_INSTALL_DIR := +endif + +libstore-test-support_SOURCES := $(wildcard $(d)/tests/*.cc) + +libstore-test-support_CXXFLAGS += $(libstore-tests_EXTRA_INCLUDES) + +libstore-test-support_LIBS = \ + libutil-test-support \ + libstore libutil + +libstore-test-support_LDFLAGS := -lrapidcheck diff --git a/tests/unit/libstore-support/tests/derived-path.cc b/tests/unit/libstore-support/tests/derived-path.cc new file mode 100644 index 000000000..091706dba --- /dev/null +++ b/tests/unit/libstore-support/tests/derived-path.cc @@ -0,0 +1,57 @@ +#include + +#include + +#include "tests/derived-path.hh" + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + return gen::just(DerivedPath::Opaque { + .path = *gen::arbitrary(), + }); +} + +Gen Arbitrary::arbitrary() +{ + return gen::just(SingleDerivedPath::Built { + .drvPath = make_ref(*gen::arbitrary()), + .output = (*gen::arbitrary()).name, + }); +} + +Gen Arbitrary::arbitrary() +{ + return gen::just(DerivedPath::Built { + .drvPath = make_ref(*gen::arbitrary()), + .outputs = *gen::arbitrary(), + }); +} + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, std::variant_size_v)) { + case 0: + return gen::just(*gen::arbitrary()); + case 1: + return gen::just(*gen::arbitrary()); + default: + assert(false); + } +} + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, std::variant_size_v)) { + case 0: + return gen::just(*gen::arbitrary()); + case 1: + return gen::just(*gen::arbitrary()); + default: + assert(false); + } +} + +} diff --git a/src/libstore/tests/derived-path.hh b/tests/unit/libstore-support/tests/derived-path.hh similarity index 100% rename from src/libstore/tests/derived-path.hh rename to tests/unit/libstore-support/tests/derived-path.hh diff --git a/src/libstore/tests/libstore.hh b/tests/unit/libstore-support/tests/libstore.hh similarity index 100% rename from src/libstore/tests/libstore.hh rename to tests/unit/libstore-support/tests/libstore.hh diff --git a/tests/unit/libstore-support/tests/outputs-spec.cc b/tests/unit/libstore-support/tests/outputs-spec.cc new file mode 100644 index 000000000..e9d602203 --- /dev/null +++ b/tests/unit/libstore-support/tests/outputs-spec.cc @@ -0,0 +1,24 @@ +#include "tests/outputs-spec.hh" + +#include + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, std::variant_size_v)) { + case 0: + return gen::just((OutputsSpec) OutputsSpec::All { }); + case 1: + return gen::just((OutputsSpec) OutputsSpec::Names { + *gen::nonEmpty(gen::container(gen::map( + gen::arbitrary(), + [](StorePathName n) { return n.name; }))), + }); + default: + assert(false); + } +} + +} diff --git a/src/libstore/tests/outputs-spec.hh b/tests/unit/libstore-support/tests/outputs-spec.hh similarity index 89% rename from src/libstore/tests/outputs-spec.hh rename to tests/unit/libstore-support/tests/outputs-spec.hh index ded331b33..f5bf9042d 100644 --- a/src/libstore/tests/outputs-spec.hh +++ b/tests/unit/libstore-support/tests/outputs-spec.hh @@ -5,7 +5,7 @@ #include -#include +#include "tests/path.hh" namespace rc { using namespace nix; diff --git a/tests/unit/libstore-support/tests/path.cc b/tests/unit/libstore-support/tests/path.cc new file mode 100644 index 000000000..e5f169e94 --- /dev/null +++ b/tests/unit/libstore-support/tests/path.cc @@ -0,0 +1,82 @@ +#include + +#include + +#include "path-regex.hh" +#include "store-api.hh" + +#include "tests/hash.hh" +#include "tests/path.hh" + +namespace nix { + +void showValue(const StorePath & p, std::ostream & os) +{ + os << p.to_string(); +} + +} + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + auto len = *gen::inRange( + 1, + StorePath::MaxPathLen - StorePath::HashLen); + + std::string pre; + pre.reserve(len); + + for (size_t c = 0; c < len; ++c) { + switch (auto i = *gen::inRange(0, 10 + 2 * 26 + 6)) { + case 0 ... 9: + pre += '0' + i; + case 10 ... 35: + pre += 'A' + (i - 10); + break; + case 36 ... 61: + pre += 'a' + (i - 36); + break; + case 62: + pre += '+'; + break; + case 63: + pre += '-'; + break; + case 64: + // names aren't permitted to start with a period, + // so just fall through to the next case here + if (c != 0) { + pre += '.'; + break; + } + case 65: + pre += '_'; + break; + case 66: + pre += '?'; + break; + case 67: + pre += '='; + break; + default: + assert(false); + } + } + + return gen::just(StorePathName { + .name = std::move(pre), + }); +} + +Gen Arbitrary::arbitrary() +{ + return gen::just(StorePath { + *gen::arbitrary(), + (*gen::arbitrary()).name, + }); +} + +} // namespace rc diff --git a/src/libstore/tests/path.hh b/tests/unit/libstore-support/tests/path.hh similarity index 82% rename from src/libstore/tests/path.hh rename to tests/unit/libstore-support/tests/path.hh index 21cb62310..4751b3373 100644 --- a/src/libstore/tests/path.hh +++ b/tests/unit/libstore-support/tests/path.hh @@ -11,6 +11,9 @@ struct StorePathName { std::string name; }; +// For rapidcheck +void showValue(const StorePath & p, std::ostream & os); + } namespace rc { diff --git a/src/libstore/tests/protocol.hh b/tests/unit/libstore-support/tests/protocol.hh similarity index 96% rename from src/libstore/tests/protocol.hh rename to tests/unit/libstore-support/tests/protocol.hh index 466032a79..3c9e52c11 100644 --- a/src/libstore/tests/protocol.hh +++ b/tests/unit/libstore-support/tests/protocol.hh @@ -12,7 +12,7 @@ namespace nix { template class ProtoTest : public CharacterizationTest, public LibStoreTest { - Path unitTestData = getUnitTestData() + "/libstore/" + protocolDir; + Path unitTestData = getUnitTestData() + "/" + protocolDir; Path goldenMaster(std::string_view testStem) const override { return unitTestData + "/" + testStem + ".bin"; diff --git a/src/libstore/tests/common-protocol.cc b/tests/unit/libstore/common-protocol.cc similarity index 100% rename from src/libstore/tests/common-protocol.cc rename to tests/unit/libstore/common-protocol.cc diff --git a/unit-test-data/libstore/common-protocol/content-address.bin b/tests/unit/libstore/data/common-protocol/content-address.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/content-address.bin rename to tests/unit/libstore/data/common-protocol/content-address.bin diff --git a/unit-test-data/libstore/common-protocol/drv-output.bin b/tests/unit/libstore/data/common-protocol/drv-output.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/drv-output.bin rename to tests/unit/libstore/data/common-protocol/drv-output.bin diff --git a/unit-test-data/libstore/common-protocol/optional-content-address.bin b/tests/unit/libstore/data/common-protocol/optional-content-address.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/optional-content-address.bin rename to tests/unit/libstore/data/common-protocol/optional-content-address.bin diff --git a/unit-test-data/libstore/common-protocol/optional-store-path.bin b/tests/unit/libstore/data/common-protocol/optional-store-path.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/optional-store-path.bin rename to tests/unit/libstore/data/common-protocol/optional-store-path.bin diff --git a/unit-test-data/libstore/common-protocol/realisation.bin b/tests/unit/libstore/data/common-protocol/realisation.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/realisation.bin rename to tests/unit/libstore/data/common-protocol/realisation.bin diff --git a/unit-test-data/libstore/common-protocol/set.bin b/tests/unit/libstore/data/common-protocol/set.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/set.bin rename to tests/unit/libstore/data/common-protocol/set.bin diff --git a/unit-test-data/libstore/common-protocol/store-path.bin b/tests/unit/libstore/data/common-protocol/store-path.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/store-path.bin rename to tests/unit/libstore/data/common-protocol/store-path.bin diff --git a/unit-test-data/libstore/common-protocol/string.bin b/tests/unit/libstore/data/common-protocol/string.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/string.bin rename to tests/unit/libstore/data/common-protocol/string.bin diff --git a/unit-test-data/libstore/common-protocol/vector.bin b/tests/unit/libstore/data/common-protocol/vector.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/vector.bin rename to tests/unit/libstore/data/common-protocol/vector.bin diff --git a/unit-test-data/libstore/derivation/bad-old-version-dyn-deps.drv b/tests/unit/libstore/data/derivation/bad-old-version-dyn-deps.drv similarity index 100% rename from unit-test-data/libstore/derivation/bad-old-version-dyn-deps.drv rename to tests/unit/libstore/data/derivation/bad-old-version-dyn-deps.drv diff --git a/unit-test-data/libstore/derivation/bad-version.drv b/tests/unit/libstore/data/derivation/bad-version.drv similarity index 100% rename from unit-test-data/libstore/derivation/bad-version.drv rename to tests/unit/libstore/data/derivation/bad-version.drv diff --git a/unit-test-data/libstore/derivation/dynDerivationDeps.drv b/tests/unit/libstore/data/derivation/dynDerivationDeps.drv similarity index 100% rename from unit-test-data/libstore/derivation/dynDerivationDeps.drv rename to tests/unit/libstore/data/derivation/dynDerivationDeps.drv diff --git a/unit-test-data/libstore/derivation/dynDerivationDeps.json b/tests/unit/libstore/data/derivation/dynDerivationDeps.json similarity index 100% rename from unit-test-data/libstore/derivation/dynDerivationDeps.json rename to tests/unit/libstore/data/derivation/dynDerivationDeps.json diff --git a/unit-test-data/libstore/derivation/output-caFixedFlat.json b/tests/unit/libstore/data/derivation/output-caFixedFlat.json similarity index 100% rename from unit-test-data/libstore/derivation/output-caFixedFlat.json rename to tests/unit/libstore/data/derivation/output-caFixedFlat.json diff --git a/unit-test-data/libstore/derivation/output-caFixedNAR.json b/tests/unit/libstore/data/derivation/output-caFixedNAR.json similarity index 100% rename from unit-test-data/libstore/derivation/output-caFixedNAR.json rename to tests/unit/libstore/data/derivation/output-caFixedNAR.json diff --git a/unit-test-data/libstore/derivation/output-caFixedText.json b/tests/unit/libstore/data/derivation/output-caFixedText.json similarity index 100% rename from unit-test-data/libstore/derivation/output-caFixedText.json rename to tests/unit/libstore/data/derivation/output-caFixedText.json diff --git a/unit-test-data/libstore/derivation/output-caFloating.json b/tests/unit/libstore/data/derivation/output-caFloating.json similarity index 100% rename from unit-test-data/libstore/derivation/output-caFloating.json rename to tests/unit/libstore/data/derivation/output-caFloating.json diff --git a/unit-test-data/libstore/derivation/output-deferred.json b/tests/unit/libstore/data/derivation/output-deferred.json similarity index 100% rename from unit-test-data/libstore/derivation/output-deferred.json rename to tests/unit/libstore/data/derivation/output-deferred.json diff --git a/unit-test-data/libstore/derivation/output-impure.json b/tests/unit/libstore/data/derivation/output-impure.json similarity index 100% rename from unit-test-data/libstore/derivation/output-impure.json rename to tests/unit/libstore/data/derivation/output-impure.json diff --git a/unit-test-data/libstore/derivation/output-inputAddressed.json b/tests/unit/libstore/data/derivation/output-inputAddressed.json similarity index 100% rename from unit-test-data/libstore/derivation/output-inputAddressed.json rename to tests/unit/libstore/data/derivation/output-inputAddressed.json diff --git a/unit-test-data/libstore/derivation/simple.drv b/tests/unit/libstore/data/derivation/simple.drv similarity index 100% rename from unit-test-data/libstore/derivation/simple.drv rename to tests/unit/libstore/data/derivation/simple.drv diff --git a/unit-test-data/libstore/derivation/simple.json b/tests/unit/libstore/data/derivation/simple.json similarity index 100% rename from unit-test-data/libstore/derivation/simple.json rename to tests/unit/libstore/data/derivation/simple.json diff --git a/unit-test-data/libstore/nar-info/impure.json b/tests/unit/libstore/data/nar-info/impure.json similarity index 100% rename from unit-test-data/libstore/nar-info/impure.json rename to tests/unit/libstore/data/nar-info/impure.json diff --git a/unit-test-data/libstore/nar-info/pure.json b/tests/unit/libstore/data/nar-info/pure.json similarity index 100% rename from unit-test-data/libstore/nar-info/pure.json rename to tests/unit/libstore/data/nar-info/pure.json diff --git a/unit-test-data/libstore/path-info/impure.json b/tests/unit/libstore/data/path-info/impure.json similarity index 100% rename from unit-test-data/libstore/path-info/impure.json rename to tests/unit/libstore/data/path-info/impure.json diff --git a/unit-test-data/libstore/path-info/pure.json b/tests/unit/libstore/data/path-info/pure.json similarity index 100% rename from unit-test-data/libstore/path-info/pure.json rename to tests/unit/libstore/data/path-info/pure.json diff --git a/unit-test-data/libstore/serve-protocol/build-result-2.2.bin b/tests/unit/libstore/data/serve-protocol/build-result-2.2.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/build-result-2.2.bin rename to tests/unit/libstore/data/serve-protocol/build-result-2.2.bin diff --git a/unit-test-data/libstore/serve-protocol/build-result-2.3.bin b/tests/unit/libstore/data/serve-protocol/build-result-2.3.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/build-result-2.3.bin rename to tests/unit/libstore/data/serve-protocol/build-result-2.3.bin diff --git a/unit-test-data/libstore/serve-protocol/build-result-2.6.bin b/tests/unit/libstore/data/serve-protocol/build-result-2.6.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/build-result-2.6.bin rename to tests/unit/libstore/data/serve-protocol/build-result-2.6.bin diff --git a/unit-test-data/libstore/serve-protocol/content-address.bin b/tests/unit/libstore/data/serve-protocol/content-address.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/content-address.bin rename to tests/unit/libstore/data/serve-protocol/content-address.bin diff --git a/unit-test-data/libstore/serve-protocol/drv-output.bin b/tests/unit/libstore/data/serve-protocol/drv-output.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/drv-output.bin rename to tests/unit/libstore/data/serve-protocol/drv-output.bin diff --git a/unit-test-data/libstore/serve-protocol/optional-content-address.bin b/tests/unit/libstore/data/serve-protocol/optional-content-address.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/optional-content-address.bin rename to tests/unit/libstore/data/serve-protocol/optional-content-address.bin diff --git a/unit-test-data/libstore/serve-protocol/optional-store-path.bin b/tests/unit/libstore/data/serve-protocol/optional-store-path.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/optional-store-path.bin rename to tests/unit/libstore/data/serve-protocol/optional-store-path.bin diff --git a/unit-test-data/libstore/serve-protocol/realisation.bin b/tests/unit/libstore/data/serve-protocol/realisation.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/realisation.bin rename to tests/unit/libstore/data/serve-protocol/realisation.bin diff --git a/unit-test-data/libstore/serve-protocol/set.bin b/tests/unit/libstore/data/serve-protocol/set.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/set.bin rename to tests/unit/libstore/data/serve-protocol/set.bin diff --git a/unit-test-data/libstore/serve-protocol/store-path.bin b/tests/unit/libstore/data/serve-protocol/store-path.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/store-path.bin rename to tests/unit/libstore/data/serve-protocol/store-path.bin diff --git a/unit-test-data/libstore/serve-protocol/string.bin b/tests/unit/libstore/data/serve-protocol/string.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/string.bin rename to tests/unit/libstore/data/serve-protocol/string.bin diff --git a/unit-test-data/libstore/serve-protocol/vector.bin b/tests/unit/libstore/data/serve-protocol/vector.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/vector.bin rename to tests/unit/libstore/data/serve-protocol/vector.bin diff --git a/unit-test-data/libstore/worker-protocol/build-result-1.27.bin b/tests/unit/libstore/data/worker-protocol/build-result-1.27.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/build-result-1.27.bin rename to tests/unit/libstore/data/worker-protocol/build-result-1.27.bin diff --git a/unit-test-data/libstore/worker-protocol/build-result-1.28.bin b/tests/unit/libstore/data/worker-protocol/build-result-1.28.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/build-result-1.28.bin rename to tests/unit/libstore/data/worker-protocol/build-result-1.28.bin diff --git a/unit-test-data/libstore/worker-protocol/build-result-1.29.bin b/tests/unit/libstore/data/worker-protocol/build-result-1.29.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/build-result-1.29.bin rename to tests/unit/libstore/data/worker-protocol/build-result-1.29.bin diff --git a/unit-test-data/libstore/worker-protocol/content-address.bin b/tests/unit/libstore/data/worker-protocol/content-address.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/content-address.bin rename to tests/unit/libstore/data/worker-protocol/content-address.bin diff --git a/unit-test-data/libstore/worker-protocol/derived-path-1.29.bin b/tests/unit/libstore/data/worker-protocol/derived-path-1.29.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/derived-path-1.29.bin rename to tests/unit/libstore/data/worker-protocol/derived-path-1.29.bin diff --git a/unit-test-data/libstore/worker-protocol/derived-path-1.30.bin b/tests/unit/libstore/data/worker-protocol/derived-path-1.30.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/derived-path-1.30.bin rename to tests/unit/libstore/data/worker-protocol/derived-path-1.30.bin diff --git a/unit-test-data/libstore/worker-protocol/drv-output.bin b/tests/unit/libstore/data/worker-protocol/drv-output.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/drv-output.bin rename to tests/unit/libstore/data/worker-protocol/drv-output.bin diff --git a/unit-test-data/libstore/worker-protocol/keyed-build-result-1.29.bin b/tests/unit/libstore/data/worker-protocol/keyed-build-result-1.29.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/keyed-build-result-1.29.bin rename to tests/unit/libstore/data/worker-protocol/keyed-build-result-1.29.bin diff --git a/unit-test-data/libstore/worker-protocol/optional-content-address.bin b/tests/unit/libstore/data/worker-protocol/optional-content-address.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/optional-content-address.bin rename to tests/unit/libstore/data/worker-protocol/optional-content-address.bin diff --git a/unit-test-data/libstore/worker-protocol/optional-store-path.bin b/tests/unit/libstore/data/worker-protocol/optional-store-path.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/optional-store-path.bin rename to tests/unit/libstore/data/worker-protocol/optional-store-path.bin diff --git a/unit-test-data/libstore/worker-protocol/optional-trusted-flag.bin b/tests/unit/libstore/data/worker-protocol/optional-trusted-flag.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/optional-trusted-flag.bin rename to tests/unit/libstore/data/worker-protocol/optional-trusted-flag.bin diff --git a/unit-test-data/libstore/worker-protocol/realisation.bin b/tests/unit/libstore/data/worker-protocol/realisation.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/realisation.bin rename to tests/unit/libstore/data/worker-protocol/realisation.bin diff --git a/unit-test-data/libstore/worker-protocol/set.bin b/tests/unit/libstore/data/worker-protocol/set.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/set.bin rename to tests/unit/libstore/data/worker-protocol/set.bin diff --git a/unit-test-data/libstore/worker-protocol/store-path.bin b/tests/unit/libstore/data/worker-protocol/store-path.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/store-path.bin rename to tests/unit/libstore/data/worker-protocol/store-path.bin diff --git a/unit-test-data/libstore/worker-protocol/string.bin b/tests/unit/libstore/data/worker-protocol/string.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/string.bin rename to tests/unit/libstore/data/worker-protocol/string.bin diff --git a/unit-test-data/libstore/worker-protocol/unkeyed-valid-path-info-1.15.bin b/tests/unit/libstore/data/worker-protocol/unkeyed-valid-path-info-1.15.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/unkeyed-valid-path-info-1.15.bin rename to tests/unit/libstore/data/worker-protocol/unkeyed-valid-path-info-1.15.bin diff --git a/unit-test-data/libstore/worker-protocol/valid-path-info-1.15.bin b/tests/unit/libstore/data/worker-protocol/valid-path-info-1.15.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/valid-path-info-1.15.bin rename to tests/unit/libstore/data/worker-protocol/valid-path-info-1.15.bin diff --git a/unit-test-data/libstore/worker-protocol/valid-path-info-1.16.bin b/tests/unit/libstore/data/worker-protocol/valid-path-info-1.16.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/valid-path-info-1.16.bin rename to tests/unit/libstore/data/worker-protocol/valid-path-info-1.16.bin diff --git a/unit-test-data/libstore/worker-protocol/vector.bin b/tests/unit/libstore/data/worker-protocol/vector.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/vector.bin rename to tests/unit/libstore/data/worker-protocol/vector.bin diff --git a/src/libstore/tests/derivation.cc b/tests/unit/libstore/derivation.cc similarity index 99% rename from src/libstore/tests/derivation.cc rename to tests/unit/libstore/derivation.cc index 7becfa5ab..a7f4488fa 100644 --- a/src/libstore/tests/derivation.cc +++ b/tests/unit/libstore/derivation.cc @@ -13,7 +13,7 @@ using nlohmann::json; class DerivationTest : public CharacterizationTest, public LibStoreTest { - Path unitTestData = getUnitTestData() + "/libstore/derivation"; + Path unitTestData = getUnitTestData() + "/derivation"; public: Path goldenMaster(std::string_view testStem) const override { diff --git a/src/libstore/tests/derived-path.cc b/tests/unit/libstore/derived-path.cc similarity index 66% rename from src/libstore/tests/derived-path.cc rename to tests/unit/libstore/derived-path.cc index 3fa3c0801..c62d79a78 100644 --- a/src/libstore/tests/derived-path.cc +++ b/tests/unit/libstore/derived-path.cc @@ -1,64 +1,11 @@ #include -#include #include #include #include "tests/derived-path.hh" #include "tests/libstore.hh" -namespace rc { -using namespace nix; - -Gen Arbitrary::arbitrary() -{ - return gen::just(DerivedPath::Opaque { - .path = *gen::arbitrary(), - }); -} - -Gen Arbitrary::arbitrary() -{ - return gen::just(SingleDerivedPath::Built { - .drvPath = make_ref(*gen::arbitrary()), - .output = (*gen::arbitrary()).name, - }); -} - -Gen Arbitrary::arbitrary() -{ - return gen::just(DerivedPath::Built { - .drvPath = make_ref(*gen::arbitrary()), - .outputs = *gen::arbitrary(), - }); -} - -Gen Arbitrary::arbitrary() -{ - switch (*gen::inRange(0, std::variant_size_v)) { - case 0: - return gen::just(*gen::arbitrary()); - case 1: - return gen::just(*gen::arbitrary()); - default: - assert(false); - } -} - -Gen Arbitrary::arbitrary() -{ - switch (*gen::inRange(0, std::variant_size_v)) { - case 0: - return gen::just(*gen::arbitrary()); - case 1: - return gen::just(*gen::arbitrary()); - default: - assert(false); - } -} - -} - namespace nix { class DerivedPathTest : public LibStoreTest diff --git a/src/libstore/tests/downstream-placeholder.cc b/tests/unit/libstore/downstream-placeholder.cc similarity index 100% rename from src/libstore/tests/downstream-placeholder.cc rename to tests/unit/libstore/downstream-placeholder.cc diff --git a/tests/unit/libstore/local.mk b/tests/unit/libstore/local.mk new file mode 100644 index 000000000..63f6d011f --- /dev/null +++ b/tests/unit/libstore/local.mk @@ -0,0 +1,31 @@ +check: libstore-tests_RUN + +programs += libstore-tests + +libstore-tests_NAME = libnixstore-tests + +libstore-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data + +libstore-tests_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libstore-tests_INSTALL_DIR := $(checkbindir) +else + libstore-tests_INSTALL_DIR := +endif + +libstore-tests_SOURCES := $(wildcard $(d)/*.cc) + +libstore-tests_EXTRA_INCLUDES = \ + -I tests/unit/libstore-support \ + -I tests/unit/libutil-support \ + -I src/libstore \ + -I src/libutil + +libstore-tests_CXXFLAGS += $(libstore-tests_EXTRA_INCLUDES) + +libstore-tests_LIBS = \ + libstore-test-support libutil-test-support \ + libstore libutil + +libstore-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) diff --git a/src/libstore/tests/machines.cc b/tests/unit/libstore/machines.cc similarity index 97% rename from src/libstore/tests/machines.cc rename to tests/unit/libstore/machines.cc index fede328ea..5b66e5a5b 100644 --- a/src/libstore/tests/machines.cc +++ b/tests/unit/libstore/machines.cc @@ -139,7 +139,7 @@ TEST(machines, getMachinesWithIncorrectFormat) { } TEST(machines, getMachinesWithCorrectFileReference) { - auto path = absPath("src/libstore/tests/test-data/machines.valid"); + auto path = absPath("tests/unit/libstore/test-data/machines.valid"); ASSERT_TRUE(pathExists(path)); settings.builders = std::string("@") + path; @@ -166,6 +166,6 @@ TEST(machines, getMachinesWithIncorrectFileReference) { } TEST(machines, getMachinesWithCorrectFileReferenceToIncorrectFile) { - settings.builders = std::string("@") + absPath("src/libstore/tests/test-data/machines.bad_format"); + settings.builders = std::string("@") + absPath("tests/unit/libstore/test-data/machines.bad_format"); EXPECT_THROW(getMachines(), FormatError); } diff --git a/src/libstore/tests/nar-info-disk-cache.cc b/tests/unit/libstore/nar-info-disk-cache.cc similarity index 100% rename from src/libstore/tests/nar-info-disk-cache.cc rename to tests/unit/libstore/nar-info-disk-cache.cc diff --git a/src/libstore/tests/nar-info.cc b/tests/unit/libstore/nar-info.cc similarity index 98% rename from src/libstore/tests/nar-info.cc rename to tests/unit/libstore/nar-info.cc index 88e6e1add..4f124e89e 100644 --- a/src/libstore/tests/nar-info.cc +++ b/tests/unit/libstore/nar-info.cc @@ -13,7 +13,7 @@ using nlohmann::json; class NarInfoTest : public CharacterizationTest, public LibStoreTest { - Path unitTestData = getUnitTestData() + "/libstore/nar-info"; + Path unitTestData = getUnitTestData() + "/nar-info"; Path goldenMaster(PathView testStem) const override { return unitTestData + "/" + testStem + ".json"; diff --git a/src/libstore/tests/outputs-spec.cc b/tests/unit/libstore/outputs-spec.cc similarity index 92% rename from src/libstore/tests/outputs-spec.cc rename to tests/unit/libstore/outputs-spec.cc index 952945185..456196be1 100644 --- a/src/libstore/tests/outputs-spec.cc +++ b/tests/unit/libstore/outputs-spec.cc @@ -1,4 +1,4 @@ -#include "outputs-spec.hh" +#include "tests/outputs-spec.hh" #include #include @@ -199,31 +199,6 @@ TEST_JSON(ExtendedOutputsSpec, names, R"(["a","b"])", (ExtendedOutputsSpec::Expl #undef TEST_JSON -} - -namespace rc { -using namespace nix; - -Gen Arbitrary::arbitrary() -{ - switch (*gen::inRange(0, std::variant_size_v)) { - case 0: - return gen::just((OutputsSpec) OutputsSpec::All { }); - case 1: - return gen::just((OutputsSpec) OutputsSpec::Names { - *gen::nonEmpty(gen::container(gen::map( - gen::arbitrary(), - [](StorePathName n) { return n.name; }))), - }); - default: - assert(false); - } -} - -} - -namespace nix { - #ifndef COVERAGE RC_GTEST_PROP( diff --git a/src/libstore/tests/path-info.cc b/tests/unit/libstore/path-info.cc similarity index 97% rename from src/libstore/tests/path-info.cc rename to tests/unit/libstore/path-info.cc index 49bf623bd..18f00ca19 100644 --- a/src/libstore/tests/path-info.cc +++ b/tests/unit/libstore/path-info.cc @@ -12,7 +12,7 @@ using nlohmann::json; class PathInfoTest : public CharacterizationTest, public LibStoreTest { - Path unitTestData = getUnitTestData() + "/libstore/path-info"; + Path unitTestData = getUnitTestData() + "/path-info"; Path goldenMaster(PathView testStem) const override { return unitTestData + "/" + testStem + ".json"; diff --git a/src/libstore/tests/path.cc b/tests/unit/libstore/path.cc similarity index 59% rename from src/libstore/tests/path.cc rename to tests/unit/libstore/path.cc index 5a84d646c..30631b5fd 100644 --- a/src/libstore/tests/path.cc +++ b/tests/unit/libstore/path.cc @@ -66,79 +66,6 @@ TEST_DO_PARSE(equals_sign, "foo=foo") #undef TEST_DO_PARSE -// For rapidcheck -void showValue(const StorePath & p, std::ostream & os) { - os << p.to_string(); -} - -} - -namespace rc { -using namespace nix; - -Gen Arbitrary::arbitrary() -{ - auto len = *gen::inRange( - 1, - StorePath::MaxPathLen - std::string_view { HASH_PART }.size()); - - std::string pre; - pre.reserve(len); - - for (size_t c = 0; c < len; ++c) { - switch (auto i = *gen::inRange(0, 10 + 2 * 26 + 6)) { - case 0 ... 9: - pre += '0' + i; - case 10 ... 35: - pre += 'A' + (i - 10); - break; - case 36 ... 61: - pre += 'a' + (i - 36); - break; - case 62: - pre += '+'; - break; - case 63: - pre += '-'; - break; - case 64: - // names aren't permitted to start with a period, - // so just fall through to the next case here - if (c != 0) { - pre += '.'; - break; - } - case 65: - pre += '_'; - break; - case 66: - pre += '?'; - break; - case 67: - pre += '='; - break; - default: - assert(false); - } - } - - return gen::just(StorePathName { - .name = std::move(pre), - }); -} - -Gen Arbitrary::arbitrary() -{ - return gen::just(StorePath { - *gen::arbitrary(), - (*gen::arbitrary()).name, - }); -} - -} // namespace rc - -namespace nix { - #ifndef COVERAGE RC_GTEST_FIXTURE_PROP( diff --git a/src/libstore/tests/references.cc b/tests/unit/libstore/references.cc similarity index 100% rename from src/libstore/tests/references.cc rename to tests/unit/libstore/references.cc diff --git a/src/libstore/tests/serve-protocol.cc b/tests/unit/libstore/serve-protocol.cc similarity index 100% rename from src/libstore/tests/serve-protocol.cc rename to tests/unit/libstore/serve-protocol.cc diff --git a/src/libstore/tests/test-data/machines.bad_format b/tests/unit/libstore/test-data/machines.bad_format similarity index 100% rename from src/libstore/tests/test-data/machines.bad_format rename to tests/unit/libstore/test-data/machines.bad_format diff --git a/src/libstore/tests/test-data/machines.valid b/tests/unit/libstore/test-data/machines.valid similarity index 100% rename from src/libstore/tests/test-data/machines.valid rename to tests/unit/libstore/test-data/machines.valid diff --git a/src/libstore/tests/worker-protocol.cc b/tests/unit/libstore/worker-protocol.cc similarity index 100% rename from src/libstore/tests/worker-protocol.cc rename to tests/unit/libstore/worker-protocol.cc diff --git a/tests/unit/libutil-support/local.mk b/tests/unit/libutil-support/local.mk new file mode 100644 index 000000000..43a1551e5 --- /dev/null +++ b/tests/unit/libutil-support/local.mk @@ -0,0 +1,19 @@ +libraries += libutil-test-support + +libutil-test-support_NAME = libnixutil-test-support + +libutil-test-support_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libutil-test-support_INSTALL_DIR := $(checklibdir) +else + libutil-test-support_INSTALL_DIR := +endif + +libutil-test-support_SOURCES := $(wildcard $(d)/tests/*.cc) + +libutil-test-support_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES) + +libutil-test-support_LIBS = libutil + +libutil-test-support_LDFLAGS := -lrapidcheck diff --git a/src/libutil/tests/characterization.hh b/tests/unit/libutil-support/tests/characterization.hh similarity index 95% rename from src/libutil/tests/characterization.hh rename to tests/unit/libutil-support/tests/characterization.hh index 6eb513d68..9d6c850f0 100644 --- a/src/libutil/tests/characterization.hh +++ b/tests/unit/libutil-support/tests/characterization.hh @@ -9,8 +9,8 @@ namespace nix { /** - * The path to the `unit-test-data` directory. See the contributing - * guide in the manual for further details. + * The path to the unit test data directory. See the contributing guide + * in the manual for further details. */ static Path getUnitTestData() { return getEnv("_NIX_TEST_UNIT_DATA").value(); diff --git a/tests/unit/libutil-support/tests/hash.cc b/tests/unit/libutil-support/tests/hash.cc new file mode 100644 index 000000000..577e9890e --- /dev/null +++ b/tests/unit/libutil-support/tests/hash.cc @@ -0,0 +1,20 @@ +#include + +#include + +#include "hash.hh" + +#include "tests/hash.hh" + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + Hash hash(htSHA1); + for (size_t i = 0; i < hash.hashSize; ++i) + hash.hash[i] = *gen::arbitrary(); + return gen::just(hash); +} + +} diff --git a/src/libutil/tests/hash.hh b/tests/unit/libutil-support/tests/hash.hh similarity index 100% rename from src/libutil/tests/hash.hh rename to tests/unit/libutil-support/tests/hash.hh diff --git a/src/libutil/tests/args.cc b/tests/unit/libutil/args.cc similarity index 98% rename from src/libutil/tests/args.cc rename to tests/unit/libutil/args.cc index bea74a8c8..950224430 100644 --- a/src/libutil/tests/args.cc +++ b/tests/unit/libutil/args.cc @@ -1,5 +1,5 @@ -#include "../args.hh" -#include "libutil/fs-sink.hh" +#include "args.hh" +#include "fs-sink.hh" #include #include @@ -165,4 +165,4 @@ RC_GTEST_PROP( #endif -} \ No newline at end of file +} diff --git a/src/libutil/tests/canon-path.cc b/tests/unit/libutil/canon-path.cc similarity index 100% rename from src/libutil/tests/canon-path.cc rename to tests/unit/libutil/canon-path.cc diff --git a/src/libutil/tests/chunked-vector.cc b/tests/unit/libutil/chunked-vector.cc similarity index 100% rename from src/libutil/tests/chunked-vector.cc rename to tests/unit/libutil/chunked-vector.cc diff --git a/src/libutil/tests/closure.cc b/tests/unit/libutil/closure.cc similarity index 100% rename from src/libutil/tests/closure.cc rename to tests/unit/libutil/closure.cc diff --git a/src/libutil/tests/compression.cc b/tests/unit/libutil/compression.cc similarity index 100% rename from src/libutil/tests/compression.cc rename to tests/unit/libutil/compression.cc diff --git a/src/libutil/tests/config.cc b/tests/unit/libutil/config.cc similarity index 100% rename from src/libutil/tests/config.cc rename to tests/unit/libutil/config.cc diff --git a/unit-test-data/libutil/git/check-data.sh b/tests/unit/libutil/data/git/check-data.sh similarity index 98% rename from unit-test-data/libutil/git/check-data.sh rename to tests/unit/libutil/data/git/check-data.sh index 68b705c95..b3f59c4f1 100644 --- a/unit-test-data/libutil/git/check-data.sh +++ b/tests/unit/libutil/data/git/check-data.sh @@ -2,7 +2,7 @@ set -eu -o pipefail -export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/git-hashing/unit-test-data +export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/git-hashing/check-data mkdir -p $TEST_ROOT repo="$TEST_ROOT/scratch" diff --git a/unit-test-data/libutil/git/hello-world-blob.bin b/tests/unit/libutil/data/git/hello-world-blob.bin similarity index 100% rename from unit-test-data/libutil/git/hello-world-blob.bin rename to tests/unit/libutil/data/git/hello-world-blob.bin diff --git a/unit-test-data/libutil/git/hello-world.bin b/tests/unit/libutil/data/git/hello-world.bin similarity index 100% rename from unit-test-data/libutil/git/hello-world.bin rename to tests/unit/libutil/data/git/hello-world.bin diff --git a/unit-test-data/libutil/git/tree.bin b/tests/unit/libutil/data/git/tree.bin similarity index 100% rename from unit-test-data/libutil/git/tree.bin rename to tests/unit/libutil/data/git/tree.bin diff --git a/unit-test-data/libutil/git/tree.txt b/tests/unit/libutil/data/git/tree.txt similarity index 100% rename from unit-test-data/libutil/git/tree.txt rename to tests/unit/libutil/data/git/tree.txt diff --git a/src/libutil/tests/git.cc b/tests/unit/libutil/git.cc similarity index 97% rename from src/libutil/tests/git.cc rename to tests/unit/libutil/git.cc index 2842ea4d0..551a2d105 100644 --- a/src/libutil/tests/git.cc +++ b/tests/unit/libutil/git.cc @@ -11,7 +11,7 @@ using namespace git; class GitTest : public CharacterizationTest { - Path unitTestData = getUnitTestData() + "/libutil/git"; + Path unitTestData = getUnitTestData() + "/git"; public: @@ -86,8 +86,8 @@ TEST_F(GitTest, blob_write) { /** * This data is for "shallow" tree tests. However, we use "real" hashes - * so that we can check our test data in the corresponding functional - * test (`git-hashing/unit-test-data`). + * so that we can check our test data in a small shell script test test + * (`tests/unit/libutil/data/git/check-data.sh`). */ const static Tree tree = { { diff --git a/src/libutil/tests/hash.cc b/tests/unit/libutil/hash.cc similarity index 92% rename from src/libutil/tests/hash.cc rename to tests/unit/libutil/hash.cc index 9a5ebbb30..92291afce 100644 --- a/src/libutil/tests/hash.cc +++ b/tests/unit/libutil/hash.cc @@ -1,12 +1,8 @@ #include -#include #include -#include -#include - -#include "tests/hash.hh" +#include "hash.hh" namespace nix { @@ -68,7 +64,6 @@ namespace nix { "7ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd" "454d4423643ce80e2a9ac94fa54ca49f"); } - TEST(hashString, testKnownSHA512Hashes2) { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"; @@ -95,16 +90,3 @@ namespace nix { ASSERT_EQ(parseHashFormatOpt("sha0042"), std::nullopt); } } - -namespace rc { -using namespace nix; - -Gen Arbitrary::arbitrary() -{ - Hash hash(htSHA1); - for (size_t i = 0; i < hash.hashSize; ++i) - hash.hash[i] = *gen::arbitrary(); - return gen::just(hash); -} - -} diff --git a/src/libutil/tests/hilite.cc b/tests/unit/libutil/hilite.cc similarity index 100% rename from src/libutil/tests/hilite.cc rename to tests/unit/libutil/hilite.cc diff --git a/src/libutil/tests/json-utils.cc b/tests/unit/libutil/json-utils.cc similarity index 100% rename from src/libutil/tests/json-utils.cc rename to tests/unit/libutil/json-utils.cc diff --git a/tests/unit/libutil/local.mk b/tests/unit/libutil/local.mk new file mode 100644 index 000000000..930efb90b --- /dev/null +++ b/tests/unit/libutil/local.mk @@ -0,0 +1,31 @@ +check: libutil-tests_RUN + +programs += libutil-tests + +libutil-tests_NAME = libnixutil-tests + +libutil-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data + +libutil-tests_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libutil-tests_INSTALL_DIR := $(checkbindir) +else + libutil-tests_INSTALL_DIR := +endif + +libutil-tests_SOURCES := $(wildcard $(d)/*.cc) + +libutil-tests_EXTRA_INCLUDES = \ + -I tests/unit/libutil-support \ + -I src/libutil + +libutil-tests_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES) + +libutil-tests_LIBS = libutil-test-support libutil + +libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) + +check: $(d)/data/git/check-data.sh.test + +$(eval $(call run-test,$(d)/data/git/check-data.sh)) diff --git a/src/libutil/tests/logging.cc b/tests/unit/libutil/logging.cc similarity index 100% rename from src/libutil/tests/logging.cc rename to tests/unit/libutil/logging.cc diff --git a/src/libutil/tests/lru-cache.cc b/tests/unit/libutil/lru-cache.cc similarity index 100% rename from src/libutil/tests/lru-cache.cc rename to tests/unit/libutil/lru-cache.cc diff --git a/src/libutil/tests/pool.cc b/tests/unit/libutil/pool.cc similarity index 100% rename from src/libutil/tests/pool.cc rename to tests/unit/libutil/pool.cc diff --git a/src/libutil/tests/references.cc b/tests/unit/libutil/references.cc similarity index 100% rename from src/libutil/tests/references.cc rename to tests/unit/libutil/references.cc diff --git a/src/libutil/tests/suggestions.cc b/tests/unit/libutil/suggestions.cc similarity index 100% rename from src/libutil/tests/suggestions.cc rename to tests/unit/libutil/suggestions.cc diff --git a/src/libutil/tests/tests.cc b/tests/unit/libutil/tests.cc similarity index 100% rename from src/libutil/tests/tests.cc rename to tests/unit/libutil/tests.cc diff --git a/src/libutil/tests/url.cc b/tests/unit/libutil/url.cc similarity index 100% rename from src/libutil/tests/url.cc rename to tests/unit/libutil/url.cc diff --git a/src/libutil/tests/xml-writer.cc b/tests/unit/libutil/xml-writer.cc similarity index 100% rename from src/libutil/tests/xml-writer.cc rename to tests/unit/libutil/xml-writer.cc From 7355a48b1a4ce2e393598c2a72ef520cba9d172d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 1 Dec 2023 15:55:18 -0500 Subject: [PATCH 036/141] flake.lock: Update Nixpkgs to fix static build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The problem was since switching to use libgit2, we had a package in our closure (`http-parser`) that was always trying to build as a shared object. Underlying Nixpkgs PR (a 23.05 backport) https://github.com/NixOS/nixpkgs/pull/271202 Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/9ba29e2346bc542e9909d1021e8fd7d4b3f64db0' (2023-11-13) → 'github:NixOS/nixpkgs/36c4ac09e9bebcec1fa7b7539cddb0c9e837409c' (2023-11-30) --- flake.lock | 8 ++++---- flake.nix | 8 +++++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index f120d3b5f..3cb9e72c9 100644 --- a/flake.lock +++ b/flake.lock @@ -50,16 +50,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1700748986, - "narHash": "sha256-/nqLrNU297h3PCw4QyDpZKZEUHmialJdZW2ceYFobds=", + "lastModified": 1701355166, + "narHash": "sha256-4V7XMI0Gd+y0zsi++cEHd99u3GNL0xSTGRmiWKzGnUQ=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "9ba29e2346bc542e9909d1021e8fd7d4b3f64db0", + "rev": "36c4ac09e9bebcec1fa7b7539cddb0c9e837409c", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-23.05-small", + "ref": "staging-23.05", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index e2e510cbc..dbd45f053 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,13 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; + # TODO Go back to nixos-23.05-small once + # https://github.com/NixOS/nixpkgs/pull/271202 is merged. + # + # Also, do not grab arbitrary further staging commits. This PR was + # carefully made to be based on release-23.05 and just contain + # rebuild-causing changes to packages that Nix actually uses. + inputs.nixpkgs.url = "github:NixOS/nixpkgs/staging-23.05"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; From 59c4c82aebb814d864548c3ad2e9128ab6e902bf Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sat, 2 Dec 2023 00:56:23 +0100 Subject: [PATCH 037/141] fix links in stores overview --- doc/manual/generate-manpage.nix | 3 ++- doc/manual/src/store/types/index.md.in | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 365422af7..ae31b2a1f 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -103,7 +103,8 @@ let ${allStores} ''; index = replaceStrings - [ "@store-types@" ] [ storesOverview ] + [ "@store-types@" "./local-store.md" "./local-daemon-store.md" ] + [ storesOverview "#local-store" "#local-daemon-store" ] details.doc; storesOverview = let diff --git a/doc/manual/src/store/types/index.md.in b/doc/manual/src/store/types/index.md.in index b4db553a2..a35161ce8 100644 --- a/doc/manual/src/store/types/index.md.in +++ b/doc/manual/src/store/types/index.md.in @@ -29,15 +29,15 @@ supported settings for each store type are documented below. The special store URL `auto` causes Nix to automatically select a store as follows: -* Use the [local store](#local-store) `/nix/store` if `/nix/var/nix` +* Use the [local store](./local-store.md) `/nix/store` if `/nix/var/nix` is writable by the current user. * Otherwise, if `/nix/var/nix/daemon-socket/socket` exists, [connect - to the Nix daemon listening on that socket](#local-daemon-store). + to the Nix daemon listening on that socket](./local-daemon-store.md). -* Otherwise, on Linux only, use the [local chroot store](#local-store) +* Otherwise, on Linux only, use the [local chroot store](./local-store.md) `~/.local/share/nix/root`, which will be created automatically if it does not exist. -* Otherwise, use the [local store](#local-store) `/nix/store`. +* Otherwise, use the [local store](./local-store.md) `/nix/store`. From 51adfb9b277fe54ad03fa2c9981585f123fcc200 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sat, 2 Dec 2023 02:21:17 +0100 Subject: [PATCH 038/141] reword documentation on settings and attributes related to substitution - add links - be more concise - clarify the distinction between `preferLocalBuild` and `allowSubstitutes` --- .../src/language/advanced-attributes.md | 23 +++++-------------- src/libstore/globals.hh | 4 +--- 2 files changed, 7 insertions(+), 20 deletions(-) diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 282b75af2..5a6c00cd4 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -257,29 +257,18 @@ Derivations can declare some infrequently used optional attributes. of the environment (typically, a few hundred kilobyte). - [`preferLocalBuild`]{#adv-attr-preferLocalBuild}\ - If this attribute is set to `true` and [distributed building is - enabled](../advanced-topics/distributed-builds.md), then, if - possible, the derivation will be built locally instead of forwarded - to a remote machine. This is appropriate for trivial builders - where the cost of doing a download or remote build would exceed - the cost of building locally. + If this attribute is set to `true` and [distributed building is enabled](../advanced-topics/distributed-builds.md), then, if possible, the derivation will be built locally instead of being forwarded to a remote machine. + This is useful for derivations that are cheapest to build locally. - [`allowSubstitutes`]{#adv-attr-allowSubstitutes}\ - If this attribute is set to `false`, then Nix will always build this - derivation; it will not try to substitute its outputs. This is - useful for very trivial derivations (such as `writeText` in Nixpkgs) - that are cheaper to build than to substitute from a binary cache. + If this attribute is set to `false`, then Nix will always build this derivation (locally or remotely); it will not try to substitute its outputs. + This is useful for derivations that are cheaper to build than to substitute. - You may disable the effects of this attibute by enabling the - `always-allow-substitutes` configuration option in Nix. + This attribute can be ignored by setting [`always-allow-substitutes`](@docroot@/command-ref/conf-file.md#conf-always-allow-substitutes) to `true`. > **Note** > - > You need to have a builder configured which satisfies the - > derivation’s `system` attribute, since the derivation cannot be - > substituted. Thus it is usually a good idea to align `system` with - > `builtins.currentSystem` when setting `allowSubstitutes` to - > `false`. For most trivial derivations this should be the case. + > If set to `false`, the [`builder`](./derivations.md#attr-builder) should be able to run on the system type specified in the [`system` attribute](./derivations.md#attr-system), since the derivation cannot be substituted. - [`__structuredAttrs`]{#adv-attr-structuredAttrs}\ If the special attribute `__structuredAttrs` is set to `true`, the other derivation diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 38b0d516c..36ba51e23 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -268,9 +268,7 @@ public: Setting alwaysAllowSubstitutes{ this, false, "always-allow-substitutes", R"( - If set to `true`, Nix will ignore the `allowSubstitutes` attribute in - derivations and always attempt to use available substituters. - For more information on `allowSubstitutes`, see [the manual chapter on advanced attributes](../language/advanced-attributes.md). + If set to `true`, Nix will ignore the [`allowSubstitutes`](@docroot@/language/advanced-attributes.md) attribute in derivations and always attempt to use [available substituters](#conf-substituters). )"}; Setting buildersUseSubstitutes{ From 24b781773f3d24b62b1c36154c36fc98417cbcdb Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sat, 2 Dec 2023 03:02:58 +0100 Subject: [PATCH 039/141] fix random docs errors remove link to the contributing guide from user documentation. it doesn't help here, and the target at first glance shows redundant information. --- src/libexpr/primops.cc | 2 +- src/libstore/globals.hh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 63c90795a..c2499bdae 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -4400,7 +4400,7 @@ void EvalState::createBaseEnv() addConstant("__currentSystem", v, { .type = nString, .doc = R"( - The value of the [`system` configuration option](@docroot@/command-ref/conf-file.md#conf-pure-eval). + The value of the [`system` configuration option](@docroot@/command-ref/conf-file.md#conf-system). It can be used to set the `system` attribute for [`builtins.derivation`](@docroot@/language/derivations.md) such that the resulting derivation can be built on the same system that evaluates the Nix expression: diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 38b0d516c..8da9e371f 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -201,7 +201,7 @@ public: Nix will only build a given [derivation](@docroot@/language/derivations.md) locally when its `system` attribute equals any of the values specified here or in [`extra-platforms`](#conf-extra-platforms). The default value is set when Nix itself is compiled for the system it will run on. - The following system types are widely used, as [Nix is actively supported on these platforms](@docroot@/contributing/hacking.md#platforms): + The following system types are widely used, as Nix is actively supported on these platforms: - `x86_64-linux` - `x86_64-darwin` @@ -761,7 +761,7 @@ public: "substituters", R"( A list of [URLs of Nix stores](@docroot@/store/types/index.md#store-url-format) to be used as substituters, separated by whitespace. - A substituter is an additional [store]{@docroot@/glossary.md##gloss-store} from which Nix can obtain [store objects](@docroot@/glossary.md#gloss-store-object) instead of building them. + A substituter is an additional [store](@docroot@/glossary.md#gloss-store) from which Nix can obtain [store objects](@docroot@/glossary.md#gloss-store-object) instead of building them. Substituters are tried based on their priority value, which each substituter can set independently. Lower value means higher priority. From 368fdb482da039fd40a1a51bbf851c54f65eb4c5 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sat, 2 Dec 2023 03:06:47 +0100 Subject: [PATCH 040/141] reword description of the `builders-use-substitutes` setting --- src/libstore/globals.hh | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 38b0d516c..dcdcd31d5 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -276,13 +276,10 @@ public: Setting buildersUseSubstitutes{ this, false, "builders-use-substitutes", R"( - If set to `true`, Nix will instruct remote build machines to use - their own binary substitutes if available. In practical terms, this - means that remote hosts will fetch as many build dependencies as - possible from their own substitutes (e.g, from `cache.nixos.org`), - instead of waiting for this host to upload them all. This can - drastically reduce build times if the network connection between - this computer and the remote build host is slow. + If set to `true`, Nix will instruct [remote build machines](#conf-builders) to use their own [`substituters`](#conf-substituters) if available. + + It means that remote build hosts will fetch as many dependencies as possible from their own substituters (e.g, from `cache.nixos.org`) instead of waiting for the local machine to upload them all. + This can drastically reduce build times if the network connection between the local machine and the remote build host is slow. )"}; Setting reservedSize{this, 8 * 1024 * 1024, "gc-reserved-space", From 2c3749a335d4462412ac73eb77a81d949e1e8ba6 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Sat, 2 Dec 2023 16:08:06 +0000 Subject: [PATCH 041/141] Fix cross builds --- package.nix | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/package.nix b/package.nix index bed77ba3b..9f30eef2f 100644 --- a/package.nix +++ b/package.nix @@ -123,6 +123,10 @@ stdenv.mkDerivation (finalAttrs: { openssl sqlite xz + + # These could be checkInputs but the configure phase fails w/o them + gtest + rapidcheck ] ++ lib.optional stdenv.isLinux libseccomp ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid @@ -137,14 +141,13 @@ stdenv.mkDerivation (finalAttrs: { doCheck = true; checkInputs = [ - gtest - rapidcheck + # see buildInputs. The configure script always wants its test libs ]; nativeCheckInputs = [ git - mercurial # FIXME: remove? only needed for tests - openssh # only needed for tests (ssh-keygen) + mercurial + openssh ]; propagatedBuildInputs = [ From ca598328085fe7a379bff8777031101fba80921b Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Sat, 2 Dec 2023 16:36:59 +0000 Subject: [PATCH 042/141] Fix coverage.nix --- coverage.nix | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/coverage.nix b/coverage.nix index 2390ef52d..f952d8b09 100644 --- a/coverage.nix +++ b/coverage.nix @@ -14,19 +14,21 @@ releaseTools.coverageAnalysis { inherit (nix) src - configureFlags - nativeBuildInputs buildInputs - #checkInputs + nativeBuildInputs + propagatedBuildInputs + configureFlags + makeFlags + installFlags + doInstallCheck + installCheckFlags + installCheckTarget ; enableParallelBuilding = true; dontInstall = false; - doInstallCheck = true; - installCheckTarget = "installcheck"; # work around buggy detection in stdenv - lcovFilter = [ "*/boost/*" "*-tab.*" ]; hardeningDisable = ["fortify"]; From 118fa9689ab0e6d12b360708177f9a1b56f3d466 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Sat, 2 Dec 2023 16:42:01 +0000 Subject: [PATCH 043/141] Create internal-api-docs.nix --- flake.nix | 17 +---------------- internal-api-docs.nix | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+), 16 deletions(-) create mode 100644 internal-api-docs.nix diff --git a/flake.nix b/flake.nix index c0841a76d..b1c3a777e 100644 --- a/flake.nix +++ b/flake.nix @@ -482,22 +482,7 @@ coverage = nixpkgsFor.x86_64-linux.native.callPackage ./coverage.nix {}; # API docs for Nix's unstable internal C++ interfaces. - internal-api-docs = nixpkgsFor.x86_64-linux.native.nix.overrideAttrs (old: { - pname = "nix-internal-api-docs"; - - configureFlags = old.configureFlags ++ [ "--enable-internal-api-docs" ]; - nativeBuildInputs = old.nativeBuildInputs ++ [ nixpkgsFor.x86_64-linux.native.doxygen ]; - - dontBuild = true; - doCheck = false; - - installTargets = [ "internal-api-html" ]; - - postInstall = '' - mkdir -p $out/nix-support - echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> $out/nix-support/hydra-build-products - ''; - }); + internal-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ./internal-api-docs.nix {}; # System tests. tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // { diff --git a/internal-api-docs.nix b/internal-api-docs.nix new file mode 100644 index 000000000..ddd3fa891 --- /dev/null +++ b/internal-api-docs.nix @@ -0,0 +1,24 @@ +{ nix +, doxygen +}: + +nix.overrideAttrs (old: { + pname = "nix-internal-api-docs"; + + configureFlags = old.configureFlags ++ [ + "--enable-internal-api-docs" + ]; + nativeBuildInputs = old.nativeBuildInputs ++ [ + doxygen + ]; + + dontBuild = true; + doCheck = false; + + installTargets = [ "internal-api-html" ]; + + postInstall = '' + mkdir -p $out/nix-support + echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> $out/nix-support/hydra-build-products + ''; +}) From 19d41fb20a45d2bf66f78813514bf5c5fd420a8b Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Sat, 2 Dec 2023 17:25:47 +0000 Subject: [PATCH 044/141] Fix stuff --- flake.nix | 230 +++--------------------------------------- package.nix | 60 ++++++----- test-nix-versions.nix | 50 +++++++++ 3 files changed, 96 insertions(+), 244 deletions(-) create mode 100644 test-nix-versions.nix diff --git a/flake.nix b/flake.nix index b1c3a777e..fbce13604 100644 --- a/flake.nix +++ b/flake.nix @@ -12,6 +12,14 @@ let inherit (nixpkgs) lib; + # Experimental fileset library: https://github.com/NixOS/nixpkgs/pull/222981 + # Not an "idiomatic" flake input because: + # - Propagation to dependent locks: https://github.com/NixOS/nix/issues/7730 + # - Subflake would download redundant and huge parent flake + # - No git tree hash support: https://github.com/NixOS/nix/issues/6044 + inherit (import (builtins.fetchTarball { url = "https://github.com/NixOS/nix/archive/1bdcd7fc8a6a40b2e805bad759b36e64e911036b.tar.gz"; sha256 = "sha256:14ljlpdsp4x7h1fkhbmc4bd3vsqnx8zdql4h3037wh09ad6a0893"; })) + fileset; + officialRelease = false; # Set to true to build the release notes for the next release. @@ -56,57 +64,6 @@ }) stdenvs); - # Experimental fileset library: https://github.com/NixOS/nixpkgs/pull/222981 - # Not an "idiomatic" flake input because: - # - Propagation to dependent locks: https://github.com/NixOS/nix/issues/7730 - # - Subflake would download redundant and huge parent flake - # - No git tree hash support: https://github.com/NixOS/nix/issues/6044 - inherit (import (builtins.fetchTarball { url = "https://github.com/NixOS/nix/archive/1bdcd7fc8a6a40b2e805bad759b36e64e911036b.tar.gz"; sha256 = "sha256:14ljlpdsp4x7h1fkhbmc4bd3vsqnx8zdql4h3037wh09ad6a0893"; })) - fileset; - - baseFiles = - # .gitignore has already been processed, so any changes in it are irrelevant - # at this point. It is not represented verbatim for test purposes because - # that would interfere with repo semantics. - fileset.fileFilter (f: f.name != ".gitignore") ./.; - - configureFiles = fileset.unions [ - ./.version - ./configure.ac - ./m4 - # TODO: do we really need README.md? It doesn't seem used in the build. - ./README.md - ]; - - topLevelBuildFiles = fileset.unions [ - ./local.mk - ./Makefile - ./Makefile.config.in - ./mk - ]; - - functionalTestFiles = fileset.unions [ - ./tests/functional - (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) - ]; - - nixSrc = fileset.toSource { - root = ./.; - fileset = fileset.intersect baseFiles (fileset.unions [ - configureFiles - topLevelBuildFiles - ./boehmgc-coroutine-sp-fallback.diff - ./doc - ./misc - ./precompiled-headers.h - ./src - ./unit-test-data - ./COPYING - ./scripts/local.mk - functionalTestFiles - ]); - }; - # Memoize nixpkgs for different platforms for efficiency. nixpkgsFor = forAllSystems (system: let @@ -131,130 +88,6 @@ cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv"); }); - commonDeps = - { pkgs - , isStatic ? pkgs.stdenv.hostPlatform.isStatic - }: - with pkgs; rec { - # Use "busybox-sandbox-shell" if present, - # if not (legacy) fallback and hope it's sufficient. - sh = pkgs.busybox-sandbox-shell or (busybox.override { - useMusl = true; - enableStatic = true; - enableMinimal = true; - extraConfig = '' - CONFIG_FEATURE_FANCY_ECHO y - CONFIG_FEATURE_SH_MATH y - CONFIG_FEATURE_SH_MATH_64 y - - CONFIG_ASH y - CONFIG_ASH_OPTIMIZE_FOR_SIZE y - - CONFIG_ASH_ALIAS y - CONFIG_ASH_BASH_COMPAT y - CONFIG_ASH_CMDCMD y - CONFIG_ASH_ECHO y - CONFIG_ASH_GETOPTS y - CONFIG_ASH_INTERNAL_GLOB y - CONFIG_ASH_JOB_CONTROL y - CONFIG_ASH_PRINTF y - CONFIG_ASH_TEST y - ''; - }); - - configureFlags = - lib.optionals stdenv.isLinux [ - "--with-boost=${boost-nix}/lib" - "--with-sandbox-shell=${sh}/bin/busybox" - ] - ++ lib.optionals (stdenv.isLinux && !(isStatic && stdenv.system == "aarch64-linux")) [ - "LDFLAGS=-fuse-ld=gold" - ]; - - testConfigureFlags = [ - "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include" - ] ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ - "--enable-install-unit-tests" - "--with-check-bin-dir=${builtins.placeholder "check"}/bin" - "--with-check-lib-dir=${builtins.placeholder "check"}/lib" - ]; - - internalApiDocsConfigureFlags = [ - "--enable-internal-api-docs" - ]; - - inherit (pkgs.buildPackages) changelog-d; - - nativeBuildDeps = - [ - buildPackages.bison - buildPackages.flex - (lib.getBin buildPackages.lowdown-nix) - buildPackages.mdbook - buildPackages.mdbook-linkcheck - buildPackages.autoconf-archive - buildPackages.autoreconfHook - buildPackages.pkg-config - - # Tests - buildPackages.git - buildPackages.mercurial # FIXME: remove? only needed for tests - buildPackages.jq # Also for custom mdBook preprocessor. - buildPackages.openssh # only needed for tests (ssh-keygen) - ] - ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)] - # Official releases don't have rl-next, so we don't need to compile a changelog - ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d - ; - - buildDeps = - [ curl - bzip2 xz brotli editline - openssl sqlite - libarchive - (pkgs.libgit2.overrideAttrs (attrs: { - src = libgit2; - version = libgit2.lastModifiedDate; - cmakeFlags = (attrs.cmakeFlags or []) ++ ["-DUSE_SSH=exec"]; - })) - boost-nix - lowdown-nix - libsodium - ] - ++ lib.optionals stdenv.isLinux [libseccomp] - ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid; - - checkDeps = [ - gtest - rapidcheck - ]; - - internalApiDocsDeps = [ - buildPackages.doxygen - ]; - - awsDeps = lib.optional (stdenv.isLinux || stdenv.isDarwin) - (aws-sdk-cpp.override { - apis = ["s3" "transfer"]; - customMemoryManagement = false; - }); - - propagatedDeps = - [ ((boehmgc.override { - enableLargeConfig = true; - }).overrideAttrs(o: { - patches = (o.patches or []) ++ [ - ./boehmgc-coroutine-sp-fallback.diff - - # https://github.com/ivmai/bdwgc/pull/586 - ./boehmgc-traceable_allocator-public.diff - ]; - }) - ) - nlohmann_json - ]; - }; - installScriptFor = systems: with nixpkgsFor.x86_64-linux.native; runCommand "installer-script" @@ -289,50 +122,11 @@ echo "file installer $out/install" >> $out/nix-support/hydra-build-products ''; - testNixVersions = pkgs: client: daemon: with commonDeps { inherit pkgs; }; with pkgs.lib; pkgs.stdenv.mkDerivation { - NIX_DAEMON_PACKAGE = daemon; - NIX_CLIENT_PACKAGE = client; - name = - "nix-tests" - + optionalString - (versionAtLeast daemon.version "2.4pre20211005" && - versionAtLeast client.version "2.4pre20211005") - "-${client.version}-against-${daemon.version}"; - inherit version; - - src = fileset.toSource { - root = ./.; - fileset = fileset.intersect baseFiles (fileset.unions [ - configureFiles - topLevelBuildFiles - functionalTestFiles - ]); + testNixVersions = pkgs: client: daemon: + pkgs.callPackage ./test-nix-versions.nix { + inherit client daemon fileset; }; - VERSION_SUFFIX = versionSuffix; - - nativeBuildInputs = nativeBuildDeps; - buildInputs = buildDeps ++ awsDeps ++ checkDeps; - propagatedBuildInputs = propagatedDeps; - - enableParallelBuilding = true; - - configureFlags = - testConfigureFlags # otherwise configure fails - ++ [ "--disable-build" ]; - dontBuild = true; - doInstallCheck = true; - - installPhase = '' - mkdir -p $out - ''; - - installCheckPhase = '' - mkdir -p src/nix-channel - make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES - ''; - }; - binaryTarball = nix: pkgs: pkgs.callPackage ./binary-tarball.nix { inherit nix; }; @@ -491,7 +285,7 @@ # on a particular version of Nixpkgs. evalNixpkgs = let - inherit (nixpkgsFor.x86_64-linux.native) runCommand nix nixpkgs-regression; + inherit (nixpkgsFor.x86_64-linux.native) runCommand nix; in runCommand "eval-nixos" { buildInputs = [ nix ]; } '' diff --git a/package.nix b/package.nix index 9f30eef2f..e4c66958b 100644 --- a/package.nix +++ b/package.nix @@ -43,6 +43,30 @@ let version = lib.fileContents ./.version + versionSuffix; canRunInstalled = stdenv.buildPlatform.canExecute stdenv.hostPlatform; + + filesets = { + baseFiles = fileset.fileFilter (f: f.name != ".gitignore") ./.; + + configureFiles = fileset.unions [ + ./.version + ./configure.ac + ./m4 + # TODO: do we really need README.md? It doesn't seem used in the build. + ./README.md + ]; + + topLevelBuildFiles = fileset.unions [ + ./local.mk + ./Makefile + ./Makefile.config.in + ./mk + ]; + + functionalTestFiles = fileset.unions [ + ./tests/functional + (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) + ]; + }; in stdenv.mkDerivation (finalAttrs: { @@ -51,33 +75,13 @@ stdenv.mkDerivation (finalAttrs: { src = let - baseFiles = fileset.fileFilter (f: f.name != ".gitignore") ./.; - configureFiles = fileset.unions [ - ./.version - ./configure.ac - ./m4 - # TODO: do we really need README.md? It doesn't seem used in the build. - ./README.md - ]; - - topLevelBuildFiles = fileset.unions [ - ./local.mk - ./Makefile - ./Makefile.config.in - ./mk - ]; - - functionalTestFiles = fileset.unions [ - ./tests/functional - (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) - ]; in fileset.toSource { root = ./.; - fileset = fileset.intersect baseFiles (fileset.unions [ - configureFiles - topLevelBuildFiles + fileset = fileset.intersect filesets.baseFiles (fileset.unions [ + filesets.configureFiles + filesets.topLevelBuildFiles ./boehmgc-coroutine-sp-fallback.diff ./doc ./misc @@ -86,7 +90,7 @@ stdenv.mkDerivation (finalAttrs: { ./unit-test-data ./COPYING ./scripts/local.mk - functionalTestFiles + filesets.functionalTestFiles ]); }; @@ -231,8 +235,12 @@ stdenv.mkDerivation (finalAttrs: { hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - passthru.perl-bindings = callPackage ./perl { - inherit fileset stdenv; + passthru ={ + inherit filesets; + + perl-bindings = callPackage ./perl { + inherit fileset stdenv; + }; }; meta.platforms = lib.platforms.unix; diff --git a/test-nix-versions.nix b/test-nix-versions.nix new file mode 100644 index 000000000..15f6cd8d0 --- /dev/null +++ b/test-nix-versions.nix @@ -0,0 +1,50 @@ +{ lib +, fileset +, stdenv +, client +, daemon +}: + +stdenv.mkDerivation { + NIX_DAEMON_PACKAGE = daemon; + NIX_CLIENT_PACKAGE = client; + name = + "nix-tests" + + lib.optionalString + (lib.versionAtLeast daemon.version "2.4pre20211005" && + lib.versionAtLeast client.version "2.4pre20211005") + "-${client.version}-against-${daemon.version}"; + + inherit (client) + version + VERSION_SUFFIX + nativeBuildInputs + buildInputs + propagatedBuildInputs + ; + + src = fileset.toSource { + root = ./.; + fileset = with client.passthru.filesets; + fileset.intersect baseFiles (fileset.unions [ + configureFiles + topLevelBuildFiles + functionalTestFiles + ]); + }; + + configureFlags = client.configureFlags # otherwise configure fails + ++ [ "--disable-build" ]; + + dontBuild = true; + doInstallCheck = true; + + installPhase = '' + mkdir -p $out + ''; + + installCheckPhase = '' + mkdir -p src/nix-channel + make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES + ''; +} From 0ca49b0c8663ae82931780ae3f1f45115b966285 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 3 Dec 2023 12:47:07 -0500 Subject: [PATCH 045/141] Add installing unit test flags --- package.nix | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/package.nix b/package.nix index e4c66958b..c1a3b9455 100644 --- a/package.nix +++ b/package.nix @@ -38,6 +38,17 @@ , sqlite , util-linux , xz + +# Configuration Options +# +# This probably seems like too many degrees of freedom, but it +# faithfully reflects how the underlying configure + make build system +# work. The top-level flake.nix will choose useful combinations. + +# Whether to install unit tests. This is useful when cross compiling +# since we cannot run them natively during the build, but can do so +# later. +, installUnitTests ? stdenv.hostPlatform != stdenv.buildPlatform }: let @@ -69,7 +80,13 @@ let }; in -stdenv.mkDerivation (finalAttrs: { +stdenv.mkDerivation (finalAttrs: let + + # Either running the unit tests during the build, or installing them + # to be run later, requiresthe unit tests to be built. + buildUnitTests = finalAttrs.doCheck || installUnitTests; + +in { pname = "nix"; inherit version; @@ -97,7 +114,7 @@ stdenv.mkDerivation (finalAttrs: { VERSION_SUFFIX = versionSuffix; outputs = [ "out" "dev" "doc" ] - ++ lib.optional (stdenv.hostPlatform != stdenv.buildPlatform) "check"; + ++ lib.optional installUnitTests "check"; nativeBuildInputs = [ bison @@ -142,7 +159,7 @@ stdenv.mkDerivation (finalAttrs: { }) ; - doCheck = true; + doCheck = stdenv.hostPlatform != stdenv.buildPlatform; checkInputs = [ # see buildInputs. The configure script always wants its test libs @@ -190,14 +207,12 @@ stdenv.mkDerivation (finalAttrs: { "LDFLAGS=-fuse-ld=gold" ++ [ "--sysconfdir=/etc" ] ++ lib.optional stdenv.hostPlatform.isStatic "--enable-embedded-sandbox-shell" - ++ [ (lib.enableFeature finalAttrs.doCheck "tests") ] - ++ lib.optionals finalAttrs.doCheck ( - [ "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include" ] - ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ - "--enable-install-unit-tests" - "--with-check-bin-dir=${builtins.placeholder "check"}/bin" - "--with-check-lib-dir=${builtins.placeholder "check"}/lib" - ]) + ++ lib.optional buildUnitTests "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include" + ++ lib.optionals installUnitTests [ + "--enable-install-unit-tests" + "--with-check-bin-dir=${builtins.placeholder "check"}/bin" + "--with-check-lib-dir=${builtins.placeholder "check"}/lib" + ] ++ lib.optional (!canRunInstalled) "--disable-doc-gen"; enableParallelBuilding = true; From ce598bae144c49c61b33cdf55679ef597ede9485 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 3 Dec 2023 14:10:09 -0500 Subject: [PATCH 046/141] WIP --- coverage.nix | 8 --- flake.nix | 7 ++- internal-api-docs.nix | 24 ------- package.nix | 143 ++++++++++++++++++++++++++++++++---------- test-nix-versions.nix | 35 ----------- 5 files changed, 117 insertions(+), 100 deletions(-) delete mode 100644 internal-api-docs.nix diff --git a/coverage.nix b/coverage.nix index f952d8b09..2c5e4a06d 100644 --- a/coverage.nix +++ b/coverage.nix @@ -26,12 +26,4 @@ releaseTools.coverageAnalysis { ; enableParallelBuilding = true; - - dontInstall = false; - - lcovFilter = [ "*/boost/*" "*-tab.*" ]; - - hardeningDisable = ["fortify"]; - - NIX_CFLAGS_COMPILE = "-DCOVERAGE=1"; } diff --git a/flake.nix b/flake.nix index fbce13604..44ce2d306 100644 --- a/flake.nix +++ b/flake.nix @@ -276,7 +276,12 @@ coverage = nixpkgsFor.x86_64-linux.native.callPackage ./coverage.nix {}; # API docs for Nix's unstable internal C++ interfaces. - internal-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ./internal-api-docs.nix {}; + internal-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ./package.nix { + doBuild = false; + doCheck = false; + doInstallCheck = false; + enableInternalAPIDocs = true; + }; # System tests. tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // { diff --git a/internal-api-docs.nix b/internal-api-docs.nix deleted file mode 100644 index ddd3fa891..000000000 --- a/internal-api-docs.nix +++ /dev/null @@ -1,24 +0,0 @@ -{ nix -, doxygen -}: - -nix.overrideAttrs (old: { - pname = "nix-internal-api-docs"; - - configureFlags = old.configureFlags ++ [ - "--enable-internal-api-docs" - ]; - nativeBuildInputs = old.nativeBuildInputs ++ [ - doxygen - ]; - - dontBuild = true; - doCheck = false; - - installTargets = [ "internal-api-html" ]; - - postInstall = '' - mkdir -p $out/nix-support - echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> $out/nix-support/hydra-build-products - ''; -}) diff --git a/package.nix b/package.nix index c1a3b9455..39fee8472 100644 --- a/package.nix +++ b/package.nix @@ -1,6 +1,7 @@ { lib , callPackage , stdenv +, releaseTools , versionSuffix ? "" , officialRelease ? false , buildUnreleasedNotes ? false @@ -21,6 +22,7 @@ , git , gtest , jq +, doxygen , libarchive , libcpuid , libgit2 @@ -45,16 +47,35 @@ # faithfully reflects how the underlying configure + make build system # work. The top-level flake.nix will choose useful combinations. +, pname ? "nix" + +, doBuild ? true +, doCheck ? stdenv.buildPlatform.canExecute stdenv.hostPlatform +, doInstallCheck ? stdenv.buildPlatform.canExecute stdenv.hostPlatform + +, withCoverageChecks ? false + +# Whether to build the internal API docs, can be done separately from +# everything else. +, enableInternalAPIDocs ? false + # Whether to install unit tests. This is useful when cross compiling # since we cannot run them natively during the build, but can do so # later. , installUnitTests ? stdenv.hostPlatform != stdenv.buildPlatform + +, test-daemon ? null +, test-client ? null }: let version = lib.fileContents ./.version + versionSuffix; canRunInstalled = stdenv.buildPlatform.canExecute stdenv.hostPlatform; + attrs = { + inherit doBuild doCheck doInstallCheck; + }; + filesets = { baseFiles = fileset.fileFilter (f: f.name != ".gitignore") ./.; @@ -78,17 +99,30 @@ let (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) ]; }; + + mkDerivation = + if withCoverageChecks + then releaseTools.coverageAnalysis + else stdenv.mkDerivation; in -stdenv.mkDerivation (finalAttrs: let +mkDerivation (finalAttrs: let + + inherit (finalAttrs) + doCheck + doInstallCheck + ; + + doBuild = !finalAttrs.dontBuild; # Either running the unit tests during the build, or installing them # to be run later, requiresthe unit tests to be built. - buildUnitTests = finalAttrs.doCheck || installUnitTests; + buildUnitTests = doCheck || installUnitTests; + + anySortOfTesting = buildUnitTests || doInstallCheck; in { - pname = "nix"; - inherit version; + inherit pname version; src = let @@ -96,9 +130,10 @@ in { in fileset.toSource { root = ./.; - fileset = fileset.intersect filesets.baseFiles (fileset.unions [ + fileset = fileset.intersect filesets.baseFiles (fileset.unions ([ filesets.configureFiles filesets.topLevelBuildFiles + ] ++ lib.optionals doBuild [ ./boehmgc-coroutine-sp-fallback.diff ./doc ./misc @@ -107,8 +142,9 @@ in { ./unit-test-data ./COPYING ./scripts/local.mk + ] ++ lib.optionals anySortOfTesting [ filesets.functionalTestFiles - ]); + ])); }; VERSION_SUFFIX = versionSuffix; @@ -159,7 +195,13 @@ in { }) ; - doCheck = stdenv.hostPlatform != stdenv.buildPlatform; + propagatedBuildInputs = [ + boehmgc + nlohmann_json + ]; + + dontBuild = !attrs.doBuild; + doCheck = attrs.doCheck; checkInputs = [ # see buildInputs. The configure script always wants its test libs @@ -169,11 +211,8 @@ in { git mercurial openssh - ]; - - propagatedBuildInputs = [ - boehmgc - nlohmann_json + ] ++ lib.optionals enableInternalAPIDocs [ + doxygen ]; disallowedReferences = [ boost ]; @@ -198,30 +237,41 @@ in { ''} ''; - configureFlags = - lib.optionals stdenv.isLinux [ - "--with-boost=${boost}/lib" - "--with-sandbox-shell=${sh}/bin/busybox" - ] - ++ lib.optional (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) - "LDFLAGS=-fuse-ld=gold" - ++ [ "--sysconfdir=/etc" ] + configureFlags = [ + "--sysconfdir=/etc" + (lib.enableFeature doBuild "build") + (lib.enableFeature anySortOfTesting "test") + (lib.enableFeature enableInternalAPIDocs "internal-api-docs") + (lib.enableFeature canRunInstalled "doc-gen") + (lib.enableFeature installUnitTests "install-unit-tests") + ] ++ lib.optionals installUnitTests [ + "--with-check-bin-dir=${builtins.placeholder "check"}/bin" + "--with-check-lib-dir=${builtins.placeholder "check"}/lib" + ] ++ lib.optionals stdenv.isLinux [ + "--with-boost=${boost}/lib" + "--with-sandbox-shell=${sh}/bin/busybox" + ] ++ lib.optional (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) + "LDFLAGS=-fuse-ld=gold" ++ lib.optional stdenv.hostPlatform.isStatic "--enable-embedded-sandbox-shell" - ++ lib.optional buildUnitTests "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include" - ++ lib.optionals installUnitTests [ - "--enable-install-unit-tests" - "--with-check-bin-dir=${builtins.placeholder "check"}/bin" - "--with-check-lib-dir=${builtins.placeholder "check"}/lib" - ] - ++ lib.optional (!canRunInstalled) "--disable-doc-gen"; + ++ lib.optional buildUnitTests "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include"; enableParallelBuilding = true; makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1"; + installTargets = lib.optional doBuild "install" + ++ lib.optional enableInternalAPIDocs "internal-api-html"; + installFlags = "sysconfdir=$(out)/etc"; - postInstall = '' + # In this case we are probably just running tests, and so there isn't + # anything to install, we just make an empty directory to signify tests + # succeeded. + installPhase = if finalAttrs.installTargets != [] then null else '' + mkdir -p $out + ''; + + postInstall = lib.optionalString doBuild '' mkdir -p $doc/nix-support echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products ${lib.optionalString stdenv.hostPlatform.isStatic '' @@ -238,19 +288,29 @@ in { $out/lib/libboost_regex.dylib \ $out/lib/libnixexpr.dylib ''} + '' + lib.optionalString enableInternalAPIDocs '' + mkdir -p $out/nix-support + echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> $out/nix-support/hydra-build-products ''; - doInstallCheck = finalAttrs.doCheck; + doInstallCheck = attrs.doInstallCheck; + installCheckFlags = "sysconfdir=$(out)/etc"; installCheckTarget = "installcheck"; # work around buggy detection in stdenv + # Needed for tests if we are not doing a build, but testing existing + # built Nix. + preInstallCheck = lib.optionalString (! doBuild) '' + mkdir -p src/nix-channel + ''; + separateDebugInfo = !stdenv.hostPlatform.isStatic; strictDeps = true; hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - passthru ={ + passthru = { inherit filesets; perl-bindings = callPackage ./perl { @@ -258,6 +318,25 @@ in { }; }; - meta.platforms = lib.platforms.unix; - meta.mainProgram = "nix"; + meta = { + platforms = lib.platforms.unix; + mainProgram = "nix"; + broken = !(lib.all (a: a) [ + (installUnitTests -> doBuild) + (doCheck -> doBuild) + ]); + }; + +} // lib.optionalAttrs withCoverageChecks { + lcovFilter = [ "*/boost/*" "*-tab.*" ]; + + hardeningDisable = ["fortify"]; + + NIX_CFLAGS_COMPILE = "-DCOVERAGE=1"; + + dontInstall = false; +} // lib.optionalAttrs (test-daemon != null) { + NIX_DAEMON_PACKAGE = test-daemon; +} // lib.optionalAttrs (test-client != null) { + NIX_CLIENT_PACKAGE = test-client; }) diff --git a/test-nix-versions.nix b/test-nix-versions.nix index 15f6cd8d0..bda4621a1 100644 --- a/test-nix-versions.nix +++ b/test-nix-versions.nix @@ -6,45 +6,10 @@ }: stdenv.mkDerivation { - NIX_DAEMON_PACKAGE = daemon; - NIX_CLIENT_PACKAGE = client; name = "nix-tests" + lib.optionalString (lib.versionAtLeast daemon.version "2.4pre20211005" && lib.versionAtLeast client.version "2.4pre20211005") "-${client.version}-against-${daemon.version}"; - - inherit (client) - version - VERSION_SUFFIX - nativeBuildInputs - buildInputs - propagatedBuildInputs - ; - - src = fileset.toSource { - root = ./.; - fileset = with client.passthru.filesets; - fileset.intersect baseFiles (fileset.unions [ - configureFiles - topLevelBuildFiles - functionalTestFiles - ]); - }; - - configureFlags = client.configureFlags # otherwise configure fails - ++ [ "--disable-build" ]; - - dontBuild = true; - doInstallCheck = true; - - installPhase = '' - mkdir -p $out - ''; - - installCheckPhase = '' - mkdir -p src/nix-channel - make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES - ''; } From 3d47e024837a4340b1a0b6b6b8114e9e9e0c38a4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 3 Dec 2023 16:48:50 -0500 Subject: [PATCH 047/141] WIP --- flake.nix | 9 +++--- package.nix | 90 +++++++++++++++++++++++++++++++---------------------- 2 files changed, 56 insertions(+), 43 deletions(-) diff --git a/flake.nix b/flake.nix index 44ce2d306..85ea1d052 100644 --- a/flake.nix +++ b/flake.nix @@ -153,7 +153,7 @@ then "" else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}"; - sh = final.busybox-sandbox-shell or (final.busybox.override { + default-busybox-sandbox-shell = final.busybox.override { useMusl = true; enableStatic = true; enableMinimal = true; @@ -175,7 +175,7 @@ CONFIG_ASH_PRINTF y CONFIG_ASH_TEST y ''; - }); + }; boehmgc = (final.boehmgc.override { enableLargeConfig = true; @@ -192,10 +192,10 @@ inherit boehmgc fileset - sh stdenv versionSuffix ; + busybox-sandbox-shell = final.busybox-sandbox-shell or default-busybox-sandbox-shell; boost = final.boost.override { enableIcu = false; }; libgit2 = final.libgit2.overrideAttrs (attrs: { src = libgit2; @@ -277,9 +277,8 @@ # API docs for Nix's unstable internal C++ interfaces. internal-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ./package.nix { + inherit fileset; doBuild = false; - doCheck = false; - doInstallCheck = false; enableInternalAPIDocs = true; }; diff --git a/package.nix b/package.nix index 39fee8472..15fe52b07 100644 --- a/package.nix +++ b/package.nix @@ -36,11 +36,12 @@ , openssl , pkg-config , rapidcheck -, sh , sqlite , util-linux , xz +, busybox-sandbox-shell ? null + # Configuration Options # # This probably seems like too many degrees of freedom, but it @@ -50,11 +51,13 @@ , pname ? "nix" , doBuild ? true -, doCheck ? stdenv.buildPlatform.canExecute stdenv.hostPlatform -, doInstallCheck ? stdenv.buildPlatform.canExecute stdenv.hostPlatform +, doCheck ? __forDefaults.canRunInstalled +, doInstallCheck ? __forDefaults.canRunInstalled , withCoverageChecks ? false +# Whether to build the regular manual +, enableManual ? __forDefaults.canRunInstalled # Whether to build the internal API docs, can be done separately from # everything else. , enableInternalAPIDocs ? false @@ -62,16 +65,26 @@ # Whether to install unit tests. This is useful when cross compiling # since we cannot run them natively during the build, but can do so # later. -, installUnitTests ? stdenv.hostPlatform != stdenv.buildPlatform +, installUnitTests ? __forDefaults.canRunInstalled +# For running the functional tests against a pre-built Nix. Probably +# want to use in conjunction with `doBuild = false;`. , test-daemon ? null , test-client ? null -}: + +# Not a real argument, just the only way to approximate let-binding some +# stuff for argument defaults. +, __forDefaults ? { + canRunInstalled = doBuild && stdenv.buildPlatform.canExecute stdenv.hostPlatform; + } +} @ attrs0: let version = lib.fileContents ./.version + versionSuffix; - canRunInstalled = stdenv.buildPlatform.canExecute stdenv.hostPlatform; + # selected attributes with defaults, will be used to define some + # things which should instead be gotten via `finalAttrs` in order to + # work with overriding. attrs = { inherit doBuild doCheck doInstallCheck; }; @@ -149,7 +162,11 @@ in { VERSION_SUFFIX = versionSuffix; - outputs = [ "out" "dev" "doc" ] + outputs = [ "out" ] + ++ lib.optional doBuild "dev" + # If we are doing just build or just docs, the one thing will use + # "out". We only need additional outputs if we are doing both. + ++ lib.optional (doBuild && (enableManual || enableInternalAPIDocs)) "doc" ++ lib.optional installUnitTests "check"; nativeBuildInputs = [ @@ -164,10 +181,11 @@ in { pkg-config ] ++ lib.optional stdenv.hostPlatform.isLinux util-linux - # Official releases don't have rl-next, so we don't need to compile a changelog + # Official releases don't have rl-next, so we don't need to compile a + # changelog ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d; - buildInputs = [ + buildInputs = lib.optionals doBuild [ boost brotli bzip2 @@ -180,19 +198,14 @@ in { openssl sqlite xz - - # These could be checkInputs but the configure phase fails w/o them - gtest - rapidcheck - ] - ++ lib.optional stdenv.isLinux libseccomp - ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid - # There have been issues building these dependencies - ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin)) - (aws-sdk-cpp.override { - apis = ["s3" "transfer"]; - customMemoryManagement = false; - }) + ] ++ lib.optional stdenv.isLinux libseccomp + ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid + # There have been issues building these dependencies + ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin)) + (aws-sdk-cpp.override { + apis = ["s3" "transfer"]; + customMemoryManagement = false; + }) ; propagatedBuildInputs = [ @@ -204,7 +217,8 @@ in { doCheck = attrs.doCheck; checkInputs = [ - # see buildInputs. The configure script always wants its test libs + gtest + rapidcheck ]; nativeCheckInputs = [ @@ -242,17 +256,17 @@ in { (lib.enableFeature doBuild "build") (lib.enableFeature anySortOfTesting "test") (lib.enableFeature enableInternalAPIDocs "internal-api-docs") - (lib.enableFeature canRunInstalled "doc-gen") + (lib.enableFeature enableManual "doc-gen") (lib.enableFeature installUnitTests "install-unit-tests") ] ++ lib.optionals installUnitTests [ "--with-check-bin-dir=${builtins.placeholder "check"}/bin" "--with-check-lib-dir=${builtins.placeholder "check"}/lib" - ] ++ lib.optionals stdenv.isLinux [ + ] ++ lib.optionals (doBuild && stdenv.isLinux) [ "--with-boost=${boost}/lib" - "--with-sandbox-shell=${sh}/bin/busybox" - ] ++ lib.optional (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) + "--with-sandbox-shell=${busybox-sandbox-shell}/bin/busybox" + ] ++ lib.optional (doBuild && stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) "LDFLAGS=-fuse-ld=gold" - ++ lib.optional stdenv.hostPlatform.isStatic "--enable-embedded-sandbox-shell" + ++ lib.optional (doBuild && stdenv.hostPlatform.isStatic) "--enable-embedded-sandbox-shell" ++ lib.optional buildUnitTests "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include"; enableParallelBuilding = true; @@ -271,14 +285,14 @@ in { mkdir -p $out ''; - postInstall = lib.optionalString doBuild '' - mkdir -p $doc/nix-support - echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products - ${lib.optionalString stdenv.hostPlatform.isStatic '' + postInstall = lib.optionalString doBuild ( + '' + mkdir -p $doc/nix-support + echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products + '' + lib.optionalString stdenv.hostPlatform.isStatic '' mkdir -p $out/nix-support echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products - ''} - ${lib.optionalString stdenv.isDarwin '' + '' + lib.optionalString stdenv.isDarwin '' install_name_tool \ -change ${boost}/lib/libboost_context.dylib \ $out/lib/libboost_context.dylib \ @@ -287,10 +301,10 @@ in { -change ${boost}/lib/libboost_regex.dylib \ $out/lib/libboost_regex.dylib \ $out/lib/libnixexpr.dylib - ''} - '' + lib.optionalString enableInternalAPIDocs '' - mkdir -p $out/nix-support - echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> $out/nix-support/hydra-build-products + '' + ) + lib.optionalString enableInternalAPIDocs '' + mkdir -p ''${!outputDoc}/nix-support + echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products ''; doInstallCheck = attrs.doInstallCheck; From c71d987553530dcf02bcd7bf4c682634d7e5b6be Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 3 Dec 2023 17:12:38 -0500 Subject: [PATCH 048/141] Fix incorrect flag name --- package.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.nix b/package.nix index d2498ade2..f688cc819 100644 --- a/package.nix +++ b/package.nix @@ -254,7 +254,7 @@ in { configureFlags = [ "--sysconfdir=/etc" (lib.enableFeature doBuild "build") - (lib.enableFeature anySortOfTesting "test") + (lib.enableFeature anySortOfTesting "tests") (lib.enableFeature enableInternalAPIDocs "internal-api-docs") (lib.enableFeature enableManual "doc-gen") (lib.enableFeature installUnitTests "install-unit-tests") From 7b51086d736f8cf983744510ff40e5afbc313079 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 3 Dec 2023 18:12:05 -0500 Subject: [PATCH 049/141] More fixes --- coverage.nix | 29 ---------------- flake.nix | 97 ++++++++++++++++++++++++++++++---------------------- package.nix | 61 +++++++++++++++++++++------------ 3 files changed, 95 insertions(+), 92 deletions(-) delete mode 100644 coverage.nix diff --git a/coverage.nix b/coverage.nix deleted file mode 100644 index 2c5e4a06d..000000000 --- a/coverage.nix +++ /dev/null @@ -1,29 +0,0 @@ -{ lib -, releaseTools -, nix -, stdenv -}: - -let - inherit (nix) version; - -in - -releaseTools.coverageAnalysis { - name = "nix-coverage-${version}"; - - inherit (nix) - src - buildInputs - nativeBuildInputs - propagatedBuildInputs - configureFlags - makeFlags - installFlags - doInstallCheck - installCheckFlags - installCheckTarget - ; - - enableParallelBuilding = true; -} diff --git a/flake.nix b/flake.nix index aafcfd71b..fab8c45be 100644 --- a/flake.nix +++ b/flake.nix @@ -123,8 +123,20 @@ ''; testNixVersions = pkgs: client: daemon: - pkgs.callPackage ./test-nix-versions.nix { - inherit client daemon fileset; + pkgs.callPackage ./package.nix { + pname = + "nix-tests" + + lib.optionalString + (lib.versionAtLeast daemon.version "2.4pre20211005" && + lib.versionAtLeast client.version "2.4pre20211005") + "-${client.version}-against-${daemon.version}"; + + inherit fileset; + + test-client = client; + test-daemon = daemon; + + doBuild = false; }; binaryTarball = nix: pkgs: pkgs.callPackage ./binary-tarball.nix { @@ -134,10 +146,6 @@ overlayFor = getStdenv: final: prev: let stdenv = getStdenv final; - - lowdown-nix = final.callPackage ./lowdown.nix { - inherit lowdown-src stdenv; - }; in { nixStable = prev.nix; @@ -145,6 +153,41 @@ # Forward from the previous stage as we don’t want it to pick the lowdown override inherit (prev) nixUnstable; + default-busybox-sandbox-shell = final.busybox.override { + useMusl = true; + enableStatic = true; + enableMinimal = true; + extraConfig = '' + CONFIG_FEATURE_FANCY_ECHO y + CONFIG_FEATURE_SH_MATH y + CONFIG_FEATURE_SH_MATH_64 y + + CONFIG_ASH y + CONFIG_ASH_OPTIMIZE_FOR_SIZE y + + CONFIG_ASH_ALIAS y + CONFIG_ASH_BASH_COMPAT y + CONFIG_ASH_CMDCMD y + CONFIG_ASH_ECHO y + CONFIG_ASH_GETOPTS y + CONFIG_ASH_INTERNAL_GLOB y + CONFIG_ASH_JOB_CONTROL y + CONFIG_ASH_PRINTF y + CONFIG_ASH_TEST y + ''; + }; + + lowdown-nix = final.callPackage ./lowdown.nix { + inherit lowdown-src stdenv; + }; + + libgit2-nix = final.libgit2.overrideAttrs (attrs: { + src = libgit2; + version = libgit2.lastModifiedDate; + cmakeFlags = attrs.cmakeFlags or [] + ++ [ "-DUSE_SSH=exec" ]; + }); + nix = let officialRelease = false; @@ -153,30 +196,6 @@ then "" else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}"; - default-busybox-sandbox-shell = final.busybox.override { - useMusl = true; - enableStatic = true; - enableMinimal = true; - extraConfig = '' - CONFIG_FEATURE_FANCY_ECHO y - CONFIG_FEATURE_SH_MATH y - CONFIG_FEATURE_SH_MATH_64 y - - CONFIG_ASH y - CONFIG_ASH_OPTIMIZE_FOR_SIZE y - - CONFIG_ASH_ALIAS y - CONFIG_ASH_BASH_COMPAT y - CONFIG_ASH_CMDCMD y - CONFIG_ASH_ECHO y - CONFIG_ASH_GETOPTS y - CONFIG_ASH_INTERNAL_GLOB y - CONFIG_ASH_JOB_CONTROL y - CONFIG_ASH_PRINTF y - CONFIG_ASH_TEST y - ''; - }; - boehmgc = (final.boehmgc.override { enableLargeConfig = true; }).overrideAttrs(o: { @@ -195,18 +214,11 @@ stdenv versionSuffix ; - busybox-sandbox-shell = final.busybox-sandbox-shell or default-busybox-sandbox-shell; - libgit2 = final.libgit2.overrideAttrs (attrs: { - src = libgit2; - version = libgit2.lastModifiedDate; - cmakeFlags = attrs.cmakeFlags or [] - ++ [ "-DUSE_SSH=exec" ]; - }); - lowdown = lowdown-nix; officialRelease = false; + libgit2 = final.libgit2-nix; + lowdown = final.lowdown-nix; + busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell; }; - - inherit lowdown-nix; }; in { @@ -272,7 +284,10 @@ dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); # Line coverage analysis. - coverage = nixpkgsFor.x86_64-linux.native.callPackage ./coverage.nix {}; + coverage = nixpkgsFor.x86_64-linux.native.nix.override { + pname = "nix-coverage"; + withCoverageChecks = true; + }; # API docs for Nix's unstable internal C++ interfaces. internal-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ./package.nix { diff --git a/package.nix b/package.nix index f688cc819..0758f989e 100644 --- a/package.nix +++ b/package.nix @@ -2,9 +2,6 @@ , callPackage , stdenv , releaseTools -, versionSuffix ? "" -, officialRelease ? false -, buildUnreleasedNotes ? false , autoconf-archive , autoreconfHook , aws-sdk-cpp @@ -43,21 +40,25 @@ , busybox-sandbox-shell ? null # Configuration Options -# +#: # This probably seems like too many degrees of freedom, but it # faithfully reflects how the underlying configure + make build system # work. The top-level flake.nix will choose useful combinations. , pname ? "nix" +, versionSuffix ? "" +, officialRelease ? false + , doBuild ? true , doCheck ? __forDefaults.canRunInstalled -, doInstallCheck ? __forDefaults.canRunInstalled +, doInstallCheck ? test-client != null || __forDefaults.canRunInstalled , withCoverageChecks ? false # Whether to build the regular manual , enableManual ? __forDefaults.canRunInstalled +, buildUnreleasedNotes ? false # Whether to build the internal API docs, can be done separately from # everything else. , enableInternalAPIDocs ? false @@ -115,7 +116,11 @@ let mkDerivation = if withCoverageChecks - then releaseTools.coverageAnalysis + then + # TODO support `finalAttrs` args function in + # `releaseTools.coverageAnalysis`. + argsFun: + releaseTools.coverageAnalysis (let args = argsFun args; in args) else stdenv.mkDerivation; in @@ -146,6 +151,7 @@ in { fileset = fileset.intersect filesets.baseFiles (fileset.unions ([ filesets.configureFiles filesets.topLevelBuildFiles + ./doc/internal-api ] ++ lib.optionals doBuild [ ./boehmgc-coroutine-sp-fallback.diff ./doc @@ -170,20 +176,24 @@ in { ++ lib.optional installUnitTests "check"; nativeBuildInputs = [ - bison - flex - (lib.getBin lowdown) - jq # Also for custom mdBook preprocessor. - mdbook - mdbook-linkcheck autoconf-archive autoreconfHook pkg-config - ] - ++ lib.optional stdenv.hostPlatform.isLinux util-linux - # Official releases don't have rl-next, so we don't need to compile a - # changelog - ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d; + ] ++ lib.optionals doBuild [ + bison + flex + ] ++ lib.optionals enableManual [ + (lib.getBin lowdown) + mdbook + mdbook-linkcheck + ] ++ lib.optionals (doInstallCheck || enableManual) [ + jq # Also for custom mdBook preprocessor. + ] ++ lib.optional stdenv.hostPlatform.isLinux util-linux + # Official releases don't have rl-next, so we don't need to compile a + # changelog + ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d + ++ lib.optional enableInternalAPIDocs doxygen + ; buildInputs = lib.optionals doBuild [ boost @@ -225,13 +235,11 @@ in { git mercurial openssh - ] ++ lib.optionals enableInternalAPIDocs [ - doxygen ]; disallowedReferences = [ boost ]; - preConfigure = lib.optionalString (! stdenv.hostPlatform.isStatic) '' + preConfigure = lib.optionalString (doBuild && ! stdenv.hostPlatform.isStatic) '' # Copy libboost_context so we don't get all of Boost in our closure. # https://github.com/NixOS/nixpkgs/issues/45462 mkdir -p $out/lib @@ -307,7 +315,14 @@ in { doInstallCheck = attrs.doInstallCheck; installCheckFlags = "sysconfdir=$(out)/etc"; - installCheckTarget = "installcheck"; # work around buggy detection in stdenv + # work around buggy detection in stdenv + installCheckTarget = "installcheck"; + + # work around weird bug where it doesn't want to do anything + installCheckPhase = if (!doBuild && doInstallCheck) then '' + mkdir -p src/nix-channel + make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES + '' else null; # Needed for tests if we are not doing a build, but testing existing # built Nix. @@ -317,7 +332,9 @@ in { separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; + # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated + # to work with `strictDeps`. + strictDeps = !withCoverageChecks; hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; From c160c6251566e758dd4d8fd409df3fa3b2f832b9 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 3 Dec 2023 18:12:22 -0500 Subject: [PATCH 050/141] Fix underlying build system so `--disable-build` works better - Internal API docs once again work - configure skips checks for a bunch of things it doesn't need --- Makefile | 2 +- configure.ac | 50 ++++++++++++++++++++++++++++---------------------- 2 files changed, 29 insertions(+), 23 deletions(-) diff --git a/Makefile b/Makefile index eea297c89..0b2b408ca 100644 --- a/Makefile +++ b/Makefile @@ -61,7 +61,7 @@ include mk/lib.mk # by the library. Rules are not "lazy" like variables, unfortunately. ifeq ($(ENABLE_BUILD), yes) $(eval $(call include-sub-makefile, doc/manual/local.mk)) -$(eval $(call include-sub-makefile, doc/internal-api/local.mk)) endif +$(eval $(call include-sub-makefile, doc/internal-api/local.mk)) GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++2a -I src diff --git a/configure.ac b/configure.ac index f8b937eb5..f9ad3c840 100644 --- a/configure.ac +++ b/configure.ac @@ -122,7 +122,6 @@ AC_PATH_PROG(flex, flex, false) AC_PATH_PROG(bison, bison, false) AC_PATH_PROG(dot, dot) AC_PATH_PROG(lsof, lsof, lsof) -NEED_PROG(jq, jq) AC_SUBST(coreutils, [$(dirname $(type -p cat))]) @@ -133,6 +132,30 @@ AC_ARG_WITH(store-dir, AS_HELP_STRING([--with-store-dir=PATH],[path of the Nix s AC_SUBST(storedir) +# Running the functional tests without building Nix is useful for testing +# different pre-built versions of Nix against each other. +AC_ARG_ENABLE(build, AS_HELP_STRING([--disable-build],[Do not build nix]), + ENABLE_BUILD=$enableval, ENABLE_BUILD=yes) +AC_SUBST(ENABLE_BUILD) + +# Building without tests is useful for bootstrapping with a smaller footprint +# or running the tests in a separate derivation. Otherwise, we do compile and +# run them. +AC_ARG_ENABLE(tests, AS_HELP_STRING([--disable-tests],[Do not build the tests]), + ENABLE_TESTS=$enableval, ENABLE_TESTS=yes) +AC_SUBST(ENABLE_TESTS) + +# Building without API docs is the default as Nix' C++ interfaces are internal and unstable. +AC_ARG_ENABLE(internal_api_docs, AS_HELP_STRING([--enable-internal-api-docs],[Build API docs for Nix's internal unstable C++ interfaces]), + internal_api_docs=$enableval, internal_api_docs=no) +AC_SUBST(internal_api_docs) + +AS_IF( + [test "$ENABLE_BUILD" == "yes" || test "$ENABLE_TEST" == "yes"], + [NEED_PROG(jq, jq)]) + +AS_IF([test "$ENABLE_BUILD" == "yes"],[ + # Look for boost, a required dependency. # Note that AX_BOOST_BASE only exports *CPP* BOOST_CPPFLAGS, no CXX flags, # and CPPFLAGS are not passed to the C++ compiler automatically. @@ -155,18 +178,6 @@ if test "x$GCC_ATOMIC_BUILTINS_NEED_LIBATOMIC" = xyes; then LDFLAGS="-latomic $LDFLAGS" fi -# Running the functional tests without building Nix is useful for testing -# different pre-built versions of Nix against each other. -AC_ARG_ENABLE(build, AS_HELP_STRING([--disable-build],[Do not build nix]), - ENABLE_BUILD=$enableval, ENABLE_BUILD=yes) -AC_SUBST(ENABLE_BUILD) -# Building without tests is useful for bootstrapping with a smaller footprint -# or running the tests in a separate derivation. Otherwise, we do compile and -# run them. -AC_ARG_ENABLE(tests, AS_HELP_STRING([--disable-tests],[Do not build the tests]), - ENABLE_TESTS=$enableval, ENABLE_TESTS=yes) -AC_SUBST(ENABLE_TESTS) - AC_ARG_ENABLE(install-unit-tests, AS_HELP_STRING([--enable-install-unit-tests],[Install the unit tests for running later (default no)]), INSTALL_UNIT_TESTS=$enableval, INSTALL_UNIT_TESTS=no) AC_SUBST(INSTALL_UNIT_TESTS) @@ -179,11 +190,6 @@ AC_ARG_WITH(check-lib-dir, AS_HELP_STRING([--with-check-lib-dir=PATH],[path to i checklibdir=$withval, checklibdir=$libdir) AC_SUBST(checklibdir) -# Building without API docs is the default as Nix' C++ interfaces are internal and unstable. -AC_ARG_ENABLE(internal_api_docs, AS_HELP_STRING([--enable-internal-api-docs],[Build API docs for Nix's internal unstable C++ interfaces]), - internal_api_docs=$enableval, internal_api_docs=no) -AC_SUBST(internal_api_docs) - # LTO is currently broken with clang for unknown reasons; ld segfaults in the llvm plugin AC_ARG_ENABLE(lto, AS_HELP_STRING([--enable-lto],[Enable LTO (only supported with GCC) [default=no]]), lto=$enableval, lto=no) @@ -310,8 +316,7 @@ if test "$gc" = yes; then AC_DEFINE(HAVE_BOEHMGC, 1, [Whether to use the Boehm garbage collector.]) fi - -if test "$ENABLE_TESTS" = yes; then +AS_IF([test "$ENABLE_TESTS" == "yes"],[ # Look for gtest. PKG_CHECK_MODULES([GTEST], [gtest_main]) @@ -338,12 +343,11 @@ AC_LINK_IFELSE([ [AC_MSG_ERROR([librapidcheck is not found.])]) AC_LANG_POP(C++) -fi +]) # Look for nlohmann/json. PKG_CHECK_MODULES([NLOHMANN_JSON], [nlohmann_json >= 3.9]) - # documentation generation switch AC_ARG_ENABLE(doc-gen, AS_HELP_STRING([--disable-doc-gen],[disable documentation generation]), doc_generate=$enableval, doc_generate=yes) @@ -388,6 +392,8 @@ if test "$embedded_sandbox_shell" = yes; then AC_DEFINE(HAVE_EMBEDDED_SANDBOX_SHELL, 1, [Include the sandbox shell in the Nix binary.]) fi +]) + # Expand all variables in config.status. test "$prefix" = NONE && prefix=$ac_default_prefix From 7a7ad7c84b4dd37331a8f8889b02c94540522dbc Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 3 Dec 2023 18:14:36 -0500 Subject: [PATCH 051/141] Remove uneeded file --- test-nix-versions.nix | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 test-nix-versions.nix diff --git a/test-nix-versions.nix b/test-nix-versions.nix deleted file mode 100644 index bda4621a1..000000000 --- a/test-nix-versions.nix +++ /dev/null @@ -1,15 +0,0 @@ -{ lib -, fileset -, stdenv -, client -, daemon -}: - -stdenv.mkDerivation { - name = - "nix-tests" - + lib.optionalString - (lib.versionAtLeast daemon.version "2.4pre20211005" && - lib.versionAtLeast client.version "2.4pre20211005") - "-${client.version}-against-${daemon.version}"; -} From e275f0adfb6b3f360f10f5adcf140c17edc58cc6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 3 Dec 2023 18:16:07 -0500 Subject: [PATCH 052/141] Move `binary-tarball.nix` to scripts dir --- flake.nix | 2 +- binary-tarball.nix => scripts/binary-tarball.nix | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) rename binary-tarball.nix => scripts/binary-tarball.nix (85%) diff --git a/flake.nix b/flake.nix index fab8c45be..5c6ad3bc7 100644 --- a/flake.nix +++ b/flake.nix @@ -139,7 +139,7 @@ doBuild = false; }; - binaryTarball = nix: pkgs: pkgs.callPackage ./binary-tarball.nix { + binaryTarball = nix: pkgs: pkgs.callPackage ./scripts/binary-tarball.nix { inherit nix; }; diff --git a/binary-tarball.nix b/scripts/binary-tarball.nix similarity index 85% rename from binary-tarball.nix rename to scripts/binary-tarball.nix index 0053abbca..32e811c94 100644 --- a/binary-tarball.nix +++ b/scripts/binary-tarball.nix @@ -21,18 +21,18 @@ in runCommand "nix-binary-tarball-${version}" env '' cp ${installerClosureInfo}/registration $TMPDIR/reginfo - cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh - substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \ + cp ${./create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh + substitute ${./install-nix-from-closure.sh} $TMPDIR/install \ --subst-var-by nix ${nix} \ --subst-var-by cacert ${cacert} - substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \ + substitute ${./install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \ --subst-var-by nix ${nix} \ --subst-var-by cacert ${cacert} - substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \ + substitute ${./install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \ --subst-var-by nix ${nix} \ --subst-var-by cacert ${cacert} - substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \ + substitute ${./install-multi-user.sh} $TMPDIR/install-multi-user \ --subst-var-by nix ${nix} \ --subst-var-by cacert ${cacert} From 60fe4ddaa1801b37a044a2c96071d96739bd26c0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 3 Dec 2023 18:17:47 -0500 Subject: [PATCH 053/141] Expose `boehmgc-nix` in overlay --- flake.nix | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/flake.nix b/flake.nix index 5c6ad3bc7..78fc88bed 100644 --- a/flake.nix +++ b/flake.nix @@ -188,6 +188,17 @@ ++ [ "-DUSE_SSH=exec" ]; }); + boehmgc-nix = (final.boehmgc.override { + enableLargeConfig = true; + }).overrideAttrs(o: { + patches = (o.patches or []) ++ [ + ./boehmgc-coroutine-sp-fallback.diff + + # https://github.com/ivmai/bdwgc/pull/586 + ./boehmgc-traceable_allocator-public.diff + ]; + }); + nix = let officialRelease = false; @@ -196,25 +207,14 @@ then "" else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}"; - boehmgc = (final.boehmgc.override { - enableLargeConfig = true; - }).overrideAttrs(o: { - patches = (o.patches or []) ++ [ - ./boehmgc-coroutine-sp-fallback.diff - - # https://github.com/ivmai/bdwgc/pull/586 - ./boehmgc-traceable_allocator-public.diff - ]; - }); - in final.callPackage ./package.nix { inherit - boehmgc fileset stdenv versionSuffix ; officialRelease = false; + boehmgc = final.boehmgc-nix; libgit2 = final.libgit2-nix; lowdown = final.lowdown-nix; busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell; From 77003a4f0c380929f18b71476b9e7f9cd4009458 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 3 Dec 2023 18:29:15 -0500 Subject: [PATCH 054/141] Factor out the installer script --- flake.nix | 43 +++++++++++-------------------------------- scripts/installer.nix | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 32 deletions(-) create mode 100644 scripts/installer.nix diff --git a/flake.nix b/flake.nix index 78fc88bed..ecd0381a2 100644 --- a/flake.nix +++ b/flake.nix @@ -89,38 +89,17 @@ }); installScriptFor = systems: - with nixpkgsFor.x86_64-linux.native; - runCommand "installer-script" - { buildInputs = [ nix ]; - } - '' - mkdir -p $out/nix-support - - # Converts /nix/store/50p3qk8k...-nix-2.4pre20201102_550e11f/bin/nix to 50p3qk8k.../bin/nix. - tarballPath() { - # Remove the store prefix - local path=''${1#${builtins.storeDir}/} - # Get the path relative to the derivation root - local rest=''${path#*/} - # Get the derivation hash - local drvHash=''${path%%-*} - echo "$drvHash/$rest" - } - - substitute ${./scripts/install.in} $out/install \ - ${pkgs.lib.concatMapStrings - (system: let - tarball = if builtins.elem system crossSystems then self.hydraJobs.binaryTarballCross.x86_64-linux.${system} else self.hydraJobs.binaryTarball.${system}; - in '' \ - --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${tarball}/*.tar.xz) \ - --replace '@tarballPath_${system}@' $(tarballPath ${tarball}/*.tar.xz) \ - '' - ) - systems - } --replace '@nixVersion@' ${version} - - echo "file installer $out/install" >> $out/nix-support/hydra-build-products - ''; + nixpkgsFor.x86_64-linux.native.callPackage ./scripts/installer.nix { + systemTarballPairs = map + (system: { + inherit system; + tarball = + if builtins.elem system crossSystems + then self.hydraJobs.binaryTarballCross.x86_64-linux.${system} + else self.hydraJobs.binaryTarball.${system}; + }) + systems; + }; testNixVersions = pkgs: client: daemon: pkgs.callPackage ./package.nix { diff --git a/scripts/installer.nix b/scripts/installer.nix new file mode 100644 index 000000000..35d2d7fe6 --- /dev/null +++ b/scripts/installer.nix @@ -0,0 +1,35 @@ +{ lib +, runCommand +, nix +, systemTarballPairs +}: + +runCommand "installer-script" { + buildInputs = [ nix ]; +} '' + mkdir -p $out/nix-support + + # Converts /nix/store/50p3qk8k...-nix-2.4pre20201102_550e11f/bin/nix to 50p3qk8k.../bin/nix. + tarballPath() { + # Remove the store prefix + local path=''${1#${builtins.storeDir}/} + # Get the path relative to the derivation root + local rest=''${path#*/} + # Get the derivation hash + local drvHash=''${path%%-*} + echo "$drvHash/$rest" + } + + substitute ${./install.in} $out/install \ + ${lib.concatMapStrings + ({ system, tarball }: + '' \ + --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${tarball}/*.tar.xz) \ + --replace '@tarballPath_${system}@' $(tarballPath ${tarball}/*.tar.xz) \ + '' + ) + systemTarballPairs + } --replace '@nixVersion@' ${nix.version} + + echo "file installer $out/install" >> $out/nix-support/hydra-build-products +'' From f58615518c1284d4dbe4655246d4d5d6e9b2befe Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 3 Dec 2023 18:39:33 -0500 Subject: [PATCH 055/141] Add documenting comments to `package.nix` --- package.nix | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/package.nix b/package.nix index 0758f989e..7f0d78b5c 100644 --- a/package.nix +++ b/package.nix @@ -43,22 +43,37 @@ #: # This probably seems like too many degrees of freedom, but it # faithfully reflects how the underlying configure + make build system -# work. The top-level flake.nix will choose useful combinations. +# work. The top-level flake.nix will choose useful combinations of these +# options to CI. , pname ? "nix" , versionSuffix ? "" , officialRelease ? false +# Whether to build Nix. Useful to skip for tasks like (a) just +# generating API docs or (b) testing existing pre-built versions of Nix , doBuild ? true + +# Run the unit tests as part of the build. See `installUnitTests` for an +# alternative to this. , doCheck ? __forDefaults.canRunInstalled + +# Run the functional tests as part of the build. , doInstallCheck ? test-client != null || __forDefaults.canRunInstalled +# Check test coverage of Nix. Probably want to use with with at least +# one of `doCHeck` or `doInstallCheck` enabled. , withCoverageChecks ? false # Whether to build the regular manual , enableManual ? __forDefaults.canRunInstalled + +# Whether to compile `rl-next.md`, the release notes for the next +# not-yet-released version of Nix in the manul, from the individual +# change log entries in the directory. , buildUnreleasedNotes ? false + # Whether to build the internal API docs, can be done separately from # everything else. , enableInternalAPIDocs ? false @@ -350,8 +365,13 @@ in { platforms = lib.platforms.unix; mainProgram = "nix"; broken = !(lib.all (a: a) [ + # We cannot run or install unit tests if we don't build them or + # Nix proper (which they depend on). (installUnitTests -> doBuild) (doCheck -> doBuild) + # We have to build the manual to build unreleased notes, as those + # are part of the manual + (buildUnreleasedNotes -> enableManual) ]); }; From a5a45e64e18de3eb827ca83c7356dc8a088be125 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 3 Dec 2023 18:45:15 -0500 Subject: [PATCH 056/141] Don't expose file sets anymore --- package.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/package.nix b/package.nix index 7f0d78b5c..52050496c 100644 --- a/package.nix +++ b/package.nix @@ -354,8 +354,6 @@ in { hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; passthru = { - inherit filesets; - perl-bindings = callPackage ./perl { inherit fileset stdenv; }; From 7e2b1cce6abec48f85c8bc056da0ca991dfe7b32 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 3 Dec 2023 18:47:54 -0500 Subject: [PATCH 057/141] Slap on `perl-bindings` in the caller The Perl bindings are not part of Nix, but a downstream package, so they don't belong in `package.nix`. They don't really belong as an attribute on `nix` either, but we can just leave that interface as is for now. --- flake.nix | 9 +++++++++ package.nix | 7 ------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/flake.nix b/flake.nix index ecd0381a2..c92f717d5 100644 --- a/flake.nix +++ b/flake.nix @@ -197,7 +197,16 @@ libgit2 = final.libgit2-nix; lowdown = final.lowdown-nix; busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell; + } // { + # this is a proper separate downstream package, but put + # here also for back compat reasons. + perl-bindings = final.nix-perl-bindings; }; + + nix-perl-bindings = final.callPackage ./perl { + inherit fileset stdenv; + }; + }; in { diff --git a/package.nix b/package.nix index 52050496c..f6219e58a 100644 --- a/package.nix +++ b/package.nix @@ -1,5 +1,4 @@ { lib -, callPackage , stdenv , releaseTools , autoconf-archive @@ -353,12 +352,6 @@ in { hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - passthru = { - perl-bindings = callPackage ./perl { - inherit fileset stdenv; - }; - }; - meta = { platforms = lib.platforms.unix; mainProgram = "nix"; From 6e0656c66c1052bcbab204140c6b3dec81f3ab15 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 3 Dec 2023 18:53:05 -0500 Subject: [PATCH 058/141] Add another configure flag assertion --- package.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/package.nix b/package.nix index f6219e58a..42f98a48c 100644 --- a/package.nix +++ b/package.nix @@ -363,6 +363,9 @@ in { # We have to build the manual to build unreleased notes, as those # are part of the manual (buildUnreleasedNotes -> enableManual) + # The build process for the manual currently requires extracting + # data from the Nix executable we are trying to document. + (enableManual -> doBuild) ]); }; From 14c26d642ebcff3fe45c8eb6719a213c63143fb3 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 3 Dec 2023 18:57:16 -0500 Subject: [PATCH 059/141] Clean up two comments --- package.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.nix b/package.nix index 42f98a48c..96b9111f8 100644 --- a/package.nix +++ b/package.nix @@ -329,10 +329,10 @@ in { doInstallCheck = attrs.doInstallCheck; installCheckFlags = "sysconfdir=$(out)/etc"; - # work around buggy detection in stdenv + # Work around buggy detection in stdenv. installCheckTarget = "installcheck"; - # work around weird bug where it doesn't want to do anything + # Work around weird bug where it doesn't think there is a Makefile. installCheckPhase = if (!doBuild && doInstallCheck) then '' mkdir -p src/nix-channel make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES From 2e5abc0fd0d5d45e125e1d981958149624268090 Mon Sep 17 00:00:00 2001 From: wh0 Date: Sun, 3 Dec 2023 17:18:58 -0800 Subject: [PATCH 060/141] tests: avoid a chroot store without sandbox support --- tests/functional/build-remote-trustless-should-fail-0.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/functional/build-remote-trustless-should-fail-0.sh b/tests/functional/build-remote-trustless-should-fail-0.sh index fad1def59..3d4a4b097 100644 --- a/tests/functional/build-remote-trustless-should-fail-0.sh +++ b/tests/functional/build-remote-trustless-should-fail-0.sh @@ -4,6 +4,7 @@ enableFeatures "daemon-trust-override" restartDaemon +requireSandboxSupport [[ $busybox =~ busybox ]] || skipTest "no busybox" unset NIX_STORE_DIR From 3c310bde2e492c2dd8bdccdfd80076231905a429 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 14 Nov 2023 11:40:56 +0100 Subject: [PATCH 061/141] reword description for the `fetch-tree` experimental feature without knowing a lot of context, it's not clear who "we" are in that text. I'm also strongly opposed to adding procedural notes into a reference manual; it just won't age well. this change leaves a factual description of the experimental feature and its purpose. --- src/libutil/experimental-features.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index e4bdb8cb3..9b46fc5b0 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -80,12 +80,11 @@ constexpr std::array xpFeatureDetails .description = R"( Enable the use of the [`fetchTree`](@docroot@/language/builtins.md#builtins-fetchTree) built-in function in the Nix language. - `fetchTree` exposes a large suite of fetching functionality in a more systematic way. + `fetchTree` exposes a generic interface for fetching remote file system trees from different types of remote sources. The [`flakes`](#xp-feature-flakes) feature flag always enables `fetch-tree`. + This built-in was previously guarded by the `flakes` experimental feature because of that overlap. - This built-in was previously guarded by the `flakes` experimental feature because of that overlap, - but since the plan is to work on stabilizing this first (due 2024 Q1), we are putting it underneath a separate feature. - Once we've made the changes we want to make, enabling just this feature will serve as a "release candidate" --- allowing users to try out the functionality we want to stabilize and not any other functionality we don't yet want to, in isolation. + Enabling just this feature serves as a "release candidate", allowing users to try it out in isolation. )", }, { From 5fe2accb754249df6cb8f840330abfcf3bd26695 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 14 Nov 2023 11:44:34 +0100 Subject: [PATCH 062/141] fix up release note --- doc/manual/src/release-notes/rl-2.19.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/release-notes/rl-2.19.md b/doc/manual/src/release-notes/rl-2.19.md index 4eecaf929..ba6eb9c64 100644 --- a/doc/manual/src/release-notes/rl-2.19.md +++ b/doc/manual/src/release-notes/rl-2.19.md @@ -18,7 +18,7 @@ - `nix-shell` shebang lines now support single-quoted arguments. - `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). - As described in the documentation for that feature, this is because we anticipate polishing it and then stabilizing it before the rest of flakes. + This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). - The interface for creating and updating lock files has been overhauled: From 823512c1e705d1fce8dfb8cde65228364c9a8045 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 22:09:10 +0000 Subject: [PATCH 063/141] Bump zeebe-io/backport-action from 2.1.1 to 2.2.0 Bumps [zeebe-io/backport-action](https://github.com/zeebe-io/backport-action) from 2.1.1 to 2.2.0. - [Release notes](https://github.com/zeebe-io/backport-action/releases) - [Commits](https://github.com/zeebe-io/backport-action/compare/v2.1.1...v2.2.0) --- updated-dependencies: - dependency-name: zeebe-io/backport-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/backport.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 975c90b91..85ddcfad3 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -21,7 +21,7 @@ jobs: fetch-depth: 0 - name: Create backport PRs # should be kept in sync with `version` - uses: zeebe-io/backport-action@v2.1.1 + uses: zeebe-io/backport-action@v2.2.0 with: # Config README: https://github.com/zeebe-io/backport-action#backport-action github_token: ${{ secrets.GITHUB_TOKEN }} From c446e5294dbc12729e7bc55ee10b40dbaeeaacf0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 22:09:14 +0000 Subject: [PATCH 064/141] Bump cachix/install-nix-action from 23 to 24 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 23 to 24. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Commits](https://github.com/cachix/install-nix-action/compare/v23...v24) --- updated-dependencies: - dependency-name: cachix/install-nix-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index afe4dc2e3..34a23b5f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v23 + - uses: cachix/install-nix-action@v24 with: # The sandbox would otherwise be disabled by default on Darwin extra_nix_config: "sandbox = true" @@ -62,7 +62,7 @@ jobs: with: fetch-depth: 0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v23 + - uses: cachix/install-nix-action@v24 with: install_url: https://releases.nixos.org/nix/nix-2.13.3/install - uses: cachix/cachix-action@v12 @@ -84,7 +84,7 @@ jobs: steps: - uses: actions/checkout@v4 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v23 + - uses: cachix/install-nix-action@v24 with: install_url: '${{needs.installer.outputs.installerURL}}' install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" @@ -114,7 +114,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v23 + - uses: cachix/install-nix-action@v24 with: install_url: https://releases.nixos.org/nix/nix-2.13.3/install - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV From e6a3cbfceb66e06184b625a3913a786f68e71a1f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 22:09:18 +0000 Subject: [PATCH 065/141] Bump cachix/cachix-action from 12 to 13 Bumps [cachix/cachix-action](https://github.com/cachix/cachix-action) from 12 to 13. - [Release notes](https://github.com/cachix/cachix-action/releases) - [Commits](https://github.com/cachix/cachix-action/compare/v12...v13) --- updated-dependencies: - dependency-name: cachix/cachix-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index afe4dc2e3..033832c9a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: # The sandbox would otherwise be disabled by default on Darwin extra_nix_config: "sandbox = true" - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/cachix-action@v12 + - uses: cachix/cachix-action@v13 if: needs.check_secrets.outputs.cachix == 'true' with: name: '${{ env.CACHIX_NAME }}' @@ -65,7 +65,7 @@ jobs: - uses: cachix/install-nix-action@v23 with: install_url: https://releases.nixos.org/nix/nix-2.13.3/install - - uses: cachix/cachix-action@v12 + - uses: cachix/cachix-action@v13 with: name: '${{ env.CACHIX_NAME }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' @@ -119,7 +119,7 @@ jobs: install_url: https://releases.nixos.org/nix/nix-2.13.3/install - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#default.version | tr -d \")" >> $GITHUB_ENV - - uses: cachix/cachix-action@v12 + - uses: cachix/cachix-action@v13 if: needs.check_secrets.outputs.cachix == 'true' with: name: '${{ env.CACHIX_NAME }}' From e488a43f457f3ef9dba92184428bbe5381fe2634 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 22:09:25 +0000 Subject: [PATCH 066/141] Bump actions/labeler from 4 to 5 Bumps [actions/labeler](https://github.com/actions/labeler) from 4 to 5. - [Release notes](https://github.com/actions/labeler/releases) - [Commits](https://github.com/actions/labeler/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/labeler dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/labels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml index d83cb4f18..34aa4e6bd 100644 --- a/.github/workflows/labels.yml +++ b/.github/workflows/labels.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest if: github.repository_owner == 'NixOS' steps: - - uses: actions/labeler@v4 + - uses: actions/labeler@v5 with: repo-token: ${{ secrets.GITHUB_TOKEN }} sync-labels: false From 345f79d01676680f2d4ef8803790896a190c855b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 Dec 2023 15:14:28 +0100 Subject: [PATCH 067/141] Check that we can't follow symlinks outside of the allowed paths --- tests/functional/restricted.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh index b8deceacc..cb83c34b1 100644 --- a/tests/functional/restricted.sh +++ b/tests/functional/restricted.sh @@ -39,6 +39,15 @@ nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I $TEST_ROOT - [[ $(nix eval --raw --impure --restrict-eval -I . --expr 'builtins.readFile "${import ./simple.nix}/hello"') == 'Hello World!' ]] +# Check that we can't follow a symlink outside of the allowed paths. +mkdir -p $TEST_ROOT/tunnel.d +ln -sfn .. $TEST_ROOT/tunnel.d/tunnel +echo foo > $TEST_ROOT/bar + +expectStderr 1 nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readFile " -I $TEST_ROOT/tunnel.d | grepQuiet "forbidden in restricted mode" + +expectStderr 1 nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readDir " -I $TEST_ROOT/tunnel.d | grepQuiet "forbidden in restricted mode" + # Check whether we can leak symlink information through directory traversal. traverseDir="$(pwd)/restricted-traverse-me" ln -sfn "$(pwd)/restricted-secret" "$(pwd)/restricted-innocent" From 733333e87db391e4f832de65f0f49f60e50c45a4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 1 Dec 2023 17:38:34 -0500 Subject: [PATCH 068/141] Including `config.h` also needs `$(buildprefix)` Per the instruction in the manual, we want to run configure in a different directory so that we can configure + build for multiple platforms. That means `config.h` will be in the build directory. This is just like `Makefile.config`, which already is used with `$(buildprefix)`. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index eea297c89..41f14ac92 100644 --- a/Makefile +++ b/Makefile @@ -64,4 +64,4 @@ $(eval $(call include-sub-makefile, doc/manual/local.mk)) $(eval $(call include-sub-makefile, doc/internal-api/local.mk)) endif -GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++2a -I src +GLOBAL_CXXFLAGS += -g -Wall -include $(buildprefix)config.h -std=c++2a -I src From 83c067c0fa0cc5a2dca440e5c986afe40b163802 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 5 Dec 2023 23:02:59 +0100 Subject: [PATCH 069/141] PosixSourceAccessor: Don't follow any symlinks All path components must not be symlinks now (so the user needs to call `resolveSymlinks()` when needed). --- src/libexpr/parser.y | 11 +++++----- src/libexpr/primops.cc | 30 ++++++++++++++-------------- src/libutil/posix-source-accessor.cc | 27 +++++++++++++++++++++---- src/libutil/posix-source-accessor.hh | 5 +++++ src/nix-env/nix-env.cc | 6 +++--- src/nix-env/user-env.cc | 2 +- tests/functional/restricted.sh | 7 +++++-- 7 files changed, 58 insertions(+), 30 deletions(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 58fc580fc..16ad8af2e 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -692,16 +692,17 @@ SourcePath resolveExprPath(SourcePath path) /* If `path' is a symlink, follow it. This is so that relative path references work. */ - while (true) { + while (!path.path.isRoot()) { // Basic cycle/depth limit to avoid infinite loops. if (++followCount >= maxFollow) throw Error("too many symbolic links encountered while traversing the path '%s'", path); - if (path.lstat().type != InputAccessor::tSymlink) break; - path = {path.accessor, CanonPath(path.readLink(), path.path.parent().value_or(CanonPath::root))}; + auto p = path.parent().resolveSymlinks() + path.baseName(); + if (p.lstat().type != InputAccessor::tSymlink) break; + path = {path.accessor, CanonPath(p.readLink(), path.path.parent().value_or(CanonPath::root))}; } /* If `path' refers to a directory, append `/default.nix'. */ - if (path.lstat().type == InputAccessor::tDirectory) + if (path.resolveSymlinks().lstat().type == InputAccessor::tDirectory) return path + "default.nix"; return path; @@ -716,7 +717,7 @@ Expr * EvalState::parseExprFromFile(const SourcePath & path) Expr * EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr & staticEnv) { - auto buffer = path.readFile(); + auto buffer = path.resolveSymlinks().readFile(); // readFile hopefully have left some extra space for terminators buffer.append("\0\0", 2); return parse(buffer.data(), buffer.size(), Pos::Origin(path), path.parent(), staticEnv); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index c442de986..f2d51f8f5 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -110,7 +110,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context) return res; } -static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v) +static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, bool resolveSymlinks = true) { NixStringContext context; @@ -120,9 +120,9 @@ static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v) if (!context.empty() && path.accessor == state.rootFS) { auto rewrites = state.realiseContext(context); auto realPath = state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context); - return {path.accessor, CanonPath(realPath)}; - } else - return path; + path = {path.accessor, CanonPath(realPath)}; + } + return resolveSymlinks ? path.resolveSymlinks() : path; } catch (Error & e) { e.addTrace(state.positions[pos], "while realising the context of path '%s'", path); throw; @@ -162,7 +162,7 @@ static void mkOutputString( argument. */ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * vScope, Value & v) { - auto path = realisePath(state, pos, vPath); + auto path = realisePath(state, pos, vPath, false); auto path2 = path.path.abs(); // FIXME @@ -1525,16 +1525,16 @@ static RegisterPrimOp primop_storePath({ static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto & arg = *args[0]; - - auto path = realisePath(state, pos, arg); - - /* SourcePath doesn't know about trailing slash. */ - auto mustBeDir = arg.type() == nString - && (arg.string_view().ends_with("/") - || arg.string_view().ends_with("/.")); - try { + auto & arg = *args[0]; + + auto path = realisePath(state, pos, arg); + + /* SourcePath doesn't know about trailing slash. */ + auto mustBeDir = arg.type() == nString + && (arg.string_view().ends_with("/") + || arg.string_view().ends_with("/.")); + auto st = path.maybeLstat(); auto exists = st && (!mustBeDir || st->type == SourceAccessor::tDirectory); v.mkBool(exists); @@ -1771,7 +1771,7 @@ static std::string_view fileTypeToString(InputAccessor::Type type) static void prim_readFileType(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto path = realisePath(state, pos, *args[0]); + auto path = realisePath(state, pos, *args[0], false); /* Retrieve the directory entry type and stringize it. */ v.mkString(fileTypeToString(path.lstat().type)); } diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index dc96f84e5..0601e6387 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -8,9 +8,9 @@ void PosixSourceAccessor::readFile( Sink & sink, std::function sizeCallback) { - // FIXME: add O_NOFOLLOW since symlinks should be resolved by the - // caller? - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + assertNoSymlinks(path); + + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC | O_NOFOLLOW); if (!fd) throw SysError("opening file '%1%'", path); @@ -42,14 +42,16 @@ void PosixSourceAccessor::readFile( bool PosixSourceAccessor::pathExists(const CanonPath & path) { + if (auto parent = path.parent()) assertNoSymlinks(*parent); return nix::pathExists(path.abs()); } std::optional PosixSourceAccessor::maybeLstat(const CanonPath & path) { + if (auto parent = path.parent()) assertNoSymlinks(*parent); struct stat st; if (::lstat(path.c_str(), &st)) { - if (errno == ENOENT) return std::nullopt; + if (errno == ENOENT || errno == ENOTDIR) return std::nullopt; throw SysError("getting status of '%s'", showPath(path)); } mtime = std::max(mtime, st.st_mtime); @@ -66,6 +68,7 @@ std::optional PosixSourceAccessor::maybeLstat(const CanonP SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & path) { + assertNoSymlinks(path); DirEntries res; for (auto & entry : nix::readDirectory(path.abs())) { std::optional type; @@ -81,6 +84,7 @@ SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & std::string PosixSourceAccessor::readLink(const CanonPath & path) { + if (auto parent = path.parent()) assertNoSymlinks(*parent); return nix::readLink(path.abs()); } @@ -89,4 +93,19 @@ std::optional PosixSourceAccessor::getPhysicalPath(const CanonPath & return path; } +void PosixSourceAccessor::assertNoSymlinks(CanonPath path) +{ + // FIXME: cache this since it potentially causes a lot of lstat calls. + while (!path.isRoot()) { + struct stat st; + if (::lstat(path.c_str(), &st)) { + if (errno != ENOENT) + throw SysError("getting status of '%s'", showPath(path)); + } + if (S_ISLNK(st.st_mode)) + throw Error("path '%s' is a symlink", showPath(path)); + path.pop(); + } +} + } diff --git a/src/libutil/posix-source-accessor.hh b/src/libutil/posix-source-accessor.hh index a45d96bf8..7189a40e5 100644 --- a/src/libutil/posix-source-accessor.hh +++ b/src/libutil/posix-source-accessor.hh @@ -29,6 +29,11 @@ struct PosixSourceAccessor : virtual SourceAccessor std::string readLink(const CanonPath & path) override; std::optional getPhysicalPath(const CanonPath & path) override; + + /** + * Throw an error if `path` or any of its ancestors are symlinks. + */ + void assertNoSymlinks(CanonPath path); }; } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 86126c7ad..e2bbd9775 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -97,7 +97,7 @@ static bool isNixExpr(const SourcePath & path, struct InputAccessor::Stat & st) { return st.type == InputAccessor::tRegular - || (st.type == InputAccessor::tDirectory && (path + "default.nix").pathExists()); + || (st.type == InputAccessor::tDirectory && (path + "default.nix").resolveSymlinks().pathExists()); } @@ -116,11 +116,11 @@ static void getAllExprs(EvalState & state, are implemented using profiles). */ if (i == "manifest.nix") continue; - SourcePath path2 = path + i; + auto path2 = (path + i).resolveSymlinks(); InputAccessor::Stat st; try { - st = path2.resolveSymlinks().lstat(); + st = path2.lstat(); } catch (Error &) { continue; // ignore dangling symlinks in ~/.nix-defexpr } diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 250224e7d..34f6bd005 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -21,7 +21,7 @@ DrvInfos queryInstalled(EvalState & state, const Path & userEnv) auto manifestFile = userEnv + "/manifest.nix"; if (pathExists(manifestFile)) { Value v; - state.evalFile(state.rootPath(CanonPath(manifestFile)), v); + state.evalFile(state.rootPath(CanonPath(manifestFile)).resolveSymlinks(), v); Bindings & bindings(*state.allocBindings(0)); getDerivations(state, v, "", bindings, elems, false); } diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh index cb83c34b1..2d6ab964b 100644 --- a/tests/functional/restricted.sh +++ b/tests/functional/restricted.sh @@ -40,13 +40,16 @@ nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I $TEST_ROOT - [[ $(nix eval --raw --impure --restrict-eval -I . --expr 'builtins.readFile "${import ./simple.nix}/hello"') == 'Hello World!' ]] # Check that we can't follow a symlink outside of the allowed paths. -mkdir -p $TEST_ROOT/tunnel.d +mkdir -p $TEST_ROOT/tunnel.d $TEST_ROOT/foo2 ln -sfn .. $TEST_ROOT/tunnel.d/tunnel echo foo > $TEST_ROOT/bar expectStderr 1 nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readFile " -I $TEST_ROOT/tunnel.d | grepQuiet "forbidden in restricted mode" -expectStderr 1 nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readDir " -I $TEST_ROOT/tunnel.d | grepQuiet "forbidden in restricted mode" +expectStderr 1 nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readDir " -I $TEST_ROOT/tunnel.d | grepQuiet "forbidden in restricted mode" + +# Reading the parents of allowed paths should show only the ancestors of the allowed paths. +[[ $(nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readDir " -I $TEST_ROOT/tunnel.d) == '{ "tunnel.d" = "directory"; }' ]] # Check whether we can leak symlink information through directory traversal. traverseDir="$(pwd)/restricted-traverse-me" From b23273f6a29c725646b3523b1c35a0ae4a84ef61 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 5 Dec 2023 18:10:37 -0500 Subject: [PATCH 070/141] Add missing `-pthread` for test support libraries This is good in general (see how the other libraries also have long had it, since 49fe9592a47e7819179c2de4fd6068e897e944c7) but in particular needed to fix the NetBSD build. --- tests/unit/libexpr-support/local.mk | 2 +- tests/unit/libstore-support/local.mk | 2 +- tests/unit/libutil-support/local.mk | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/libexpr-support/local.mk b/tests/unit/libexpr-support/local.mk index 28e87b8f2..12a76206a 100644 --- a/tests/unit/libexpr-support/local.mk +++ b/tests/unit/libexpr-support/local.mk @@ -20,4 +20,4 @@ libexpr-test-support_LIBS = \ libstore-test-support libutil-test-support \ libexpr libstore libutil -libexpr-test-support_LDFLAGS := -lrapidcheck +libexpr-test-support_LDFLAGS := -pthread -lrapidcheck diff --git a/tests/unit/libstore-support/local.mk b/tests/unit/libstore-support/local.mk index d5d657c91..ff075c96a 100644 --- a/tests/unit/libstore-support/local.mk +++ b/tests/unit/libstore-support/local.mk @@ -18,4 +18,4 @@ libstore-test-support_LIBS = \ libutil-test-support \ libstore libutil -libstore-test-support_LDFLAGS := -lrapidcheck +libstore-test-support_LDFLAGS := -pthread -lrapidcheck diff --git a/tests/unit/libutil-support/local.mk b/tests/unit/libutil-support/local.mk index 43a1551e5..2ee2cdb6c 100644 --- a/tests/unit/libutil-support/local.mk +++ b/tests/unit/libutil-support/local.mk @@ -16,4 +16,4 @@ libutil-test-support_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES) libutil-test-support_LIBS = libutil -libutil-test-support_LDFLAGS := -lrapidcheck +libutil-test-support_LDFLAGS := -pthread -lrapidcheck From 504e4fc4576dc6a4cd5c083a3bf7b80dfb0ca220 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 Dec 2023 13:45:59 +0100 Subject: [PATCH 071/141] CanonPath: Support std::hash --- src/libfetchers/git-utils.cc | 2 +- src/libutil/canon-path.hh | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 19eae0e1d..5f2a7a8bc 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -554,7 +554,7 @@ struct GitInputAccessor : InputAccessor return toHash(*git_tree_entry_id(entry)); } - std::map lookupCache; + std::unordered_map lookupCache; /* Recursively look up 'path' relative to the root. */ git_tree_entry * lookup(const CanonPath & path) diff --git a/src/libutil/canon-path.hh b/src/libutil/canon-path.hh index 6d0519f4f..6aff4ec0d 100644 --- a/src/libutil/canon-path.hh +++ b/src/libutil/canon-path.hh @@ -205,8 +205,19 @@ public: * `CanonPath(this.makeRelative(x), this) == path`. */ std::string makeRelative(const CanonPath & path) const; + + friend class std::hash; }; std::ostream & operator << (std::ostream & stream, const CanonPath & path); } + +template<> +struct std::hash +{ + std::size_t operator ()(const nix::CanonPath & s) const noexcept + { + return std::hash{}(s.path); + } +}; From 57246c4c3802920e6167fd540dae2a0abca97f15 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 Dec 2023 13:55:07 +0100 Subject: [PATCH 072/141] PosixSourceAccessor: Cache lstat() calls Since we're doing a lot of them in assertNoSymlinks(). --- src/libutil/posix-source-accessor.cc | 56 +++++++++++++++++++--------- src/libutil/posix-source-accessor.hh | 4 ++ 2 files changed, 42 insertions(+), 18 deletions(-) diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 0601e6387..15ff76e59 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -1,5 +1,8 @@ #include "posix-source-accessor.hh" #include "signals.hh" +#include "sync.hh" + +#include namespace nix { @@ -46,23 +49,45 @@ bool PosixSourceAccessor::pathExists(const CanonPath & path) return nix::pathExists(path.abs()); } +std::optional PosixSourceAccessor::cachedLstat(const CanonPath & path) +{ + static Sync>> _cache; + + { + auto cache(_cache.lock()); + auto i = cache->find(path); + if (i != cache->end()) return i->second; + } + + std::optional st{std::in_place}; + if (::lstat(path.c_str(), &*st)) { + if (errno == ENOENT || errno == ENOTDIR) + st.reset(); + else + throw SysError("getting status of '%s'", showPath(path)); + } + + auto cache(_cache.lock()); + if (cache->size() >= 16384) cache->clear(); + cache->emplace(path, st); + + return st; +} + std::optional PosixSourceAccessor::maybeLstat(const CanonPath & path) { if (auto parent = path.parent()) assertNoSymlinks(*parent); - struct stat st; - if (::lstat(path.c_str(), &st)) { - if (errno == ENOENT || errno == ENOTDIR) return std::nullopt; - throw SysError("getting status of '%s'", showPath(path)); - } - mtime = std::max(mtime, st.st_mtime); + auto st = cachedLstat(path); + if (!st) return std::nullopt; + mtime = std::max(mtime, st->st_mtime); return Stat { .type = - S_ISREG(st.st_mode) ? tRegular : - S_ISDIR(st.st_mode) ? tDirectory : - S_ISLNK(st.st_mode) ? tSymlink : + S_ISREG(st->st_mode) ? tRegular : + S_ISDIR(st->st_mode) ? tDirectory : + S_ISLNK(st->st_mode) ? tSymlink : tMisc, - .fileSize = S_ISREG(st.st_mode) ? std::optional(st.st_size) : std::nullopt, - .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR, + .fileSize = S_ISREG(st->st_mode) ? std::optional(st->st_size) : std::nullopt, + .isExecutable = S_ISREG(st->st_mode) && st->st_mode & S_IXUSR, }; } @@ -95,14 +120,9 @@ std::optional PosixSourceAccessor::getPhysicalPath(const CanonPath & void PosixSourceAccessor::assertNoSymlinks(CanonPath path) { - // FIXME: cache this since it potentially causes a lot of lstat calls. while (!path.isRoot()) { - struct stat st; - if (::lstat(path.c_str(), &st)) { - if (errno != ENOENT) - throw SysError("getting status of '%s'", showPath(path)); - } - if (S_ISLNK(st.st_mode)) + auto st = cachedLstat(path); + if (st && S_ISLNK(st->st_mode)) throw Error("path '%s' is a symlink", showPath(path)); path.pop(); } diff --git a/src/libutil/posix-source-accessor.hh b/src/libutil/posix-source-accessor.hh index 7189a40e5..b2bd39805 100644 --- a/src/libutil/posix-source-accessor.hh +++ b/src/libutil/posix-source-accessor.hh @@ -30,10 +30,14 @@ struct PosixSourceAccessor : virtual SourceAccessor std::optional getPhysicalPath(const CanonPath & path) override; +private: + /** * Throw an error if `path` or any of its ancestors are symlinks. */ void assertNoSymlinks(CanonPath path); + + std::optional cachedLstat(const CanonPath & path); }; } From 53ab5d87c2eef72202bd76eb43e072636bbc72e8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 Dec 2023 14:05:32 +0100 Subject: [PATCH 073/141] Use expectStderr --- tests/functional/restricted.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh index 2d6ab964b..3de26eb36 100644 --- a/tests/functional/restricted.sh +++ b/tests/functional/restricted.sh @@ -14,7 +14,7 @@ nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I sr (! nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel') nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel' -I src=../../src -(! nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile ') +expectStderr 1 nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile ' | grepQuiet "forbidden in restricted mode" nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile ' -I src=. p=$(nix eval --raw --expr "builtins.fetchurl file://$(pwd)/restricted.sh" --impure --restrict-eval --allowed-uris "file://$(pwd)") From ee8540ae9055791cfec4cbf8cb6335368b867acc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= <7226587+thufschmitt@users.noreply.github.com> Date: Wed, 6 Dec 2023 14:07:08 +0100 Subject: [PATCH 074/141] Fix the labeler.yml config file labeler 5.0 changed the configuration file in a non-backwards-compatible way (https://github.com/actions/labeler/tree/main#breaking-changes-in-v5), so update our config file to match that (because all the CIs are red otherwise :grimacing: ). --- .github/labeler.yml | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/.github/labeler.yml b/.github/labeler.yml index 7544f07a6..b1b18c488 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -1,23 +1,30 @@ "documentation": - - doc/manual/* - - src/nix/**/*.md + - changed-files: + - any-glob-to-any-file: "doc/manual/*" + - any-glob-to-any-file: "src/nix/**/*.md" "store": - - src/libstore/store-api.* - - src/libstore/*-store.* + - changed-files: + - any-glob-to-any-file: "src/libstore/store-api.*" + - any-glob-to-any-file: "src/libstore/*-store.*" "fetching": - - src/libfetchers/**/* + - changed-files: + - any-glob-to-any-file: "src/libfetchers/**/*" "repl": - - src/libcmd/repl.* - - src/nix/repl.* + - changed-files: + - any-glob-to-any-file: "src/libcmd/repl.*" + - any-glob-to-any-file: "src/nix/repl.*" "new-cli": - - src/nix/**/* + - changed-files: + - any-glob-to-any-file: "src/nix/**/*" "with-tests": - # Unit tests - - src/*/tests/**/* - # Functional and integration tests - - tests/functional/**/* + - changed-files: + # Unit tests + - any-glob-to-any-file: "src/*/tests/**/*" + # Functional and integration tests + - any-glob-to-any-file: "tests/functional/**/*" + From 2bd83225004012af97d2d5977dc1de952f60aa8d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 6 Dec 2023 14:08:40 +0100 Subject: [PATCH 075/141] Update src/libfetchers/filtering-input-accessor.hh Co-authored-by: Robert Hensing --- src/libfetchers/filtering-input-accessor.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libfetchers/filtering-input-accessor.hh b/src/libfetchers/filtering-input-accessor.hh index 209d26974..e1b83c929 100644 --- a/src/libfetchers/filtering-input-accessor.hh +++ b/src/libfetchers/filtering-input-accessor.hh @@ -13,8 +13,8 @@ typedef std::function MakeNotAllowe /** * An abstract wrapping `InputAccessor` that performs access - * control. Subclasses should override `checkAccess()` to implement an - * access control policy. + * control. Subclasses should override `isAllowed()` to implement an + * access control policy. The error message is customized at construction. */ struct FilteringInputAccessor : InputAccessor { From 7fff625e39fa6b11c4c61eeacadc70a0253bdab6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= <7226587+thufschmitt@users.noreply.github.com> Date: Wed, 6 Dec 2023 14:13:45 +0100 Subject: [PATCH 076/141] =?UTF-8?q?Improve=20the=20error=20message=20for?= =?UTF-8?q?=20=E2=80=9Cmulticommands=E2=80=9D=20commands=20(#9510)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Factor out the default `MultiCommand` behavior All the `MultiCommand`s had (nearly) the same behavior when called without a subcommand. Factor out this behavior into the `NixMultiCommand` class. * Display the list of available subcommands when none is specified Whenever a user runs a command that excepts a subcommand, add the list of available subcommands to the error message. * Print the multi-command lists as Markdown lists This takes more screen real estate, but is also much more readable than a comma-separated list --- src/libcmd/command.cc | 14 ++++++++++++++ src/libcmd/command.hh | 6 +++++- src/libutil/args.cc | 5 +++-- src/libutil/args.hh | 9 ++++++--- src/nix/config.cc | 11 ++--------- src/nix/derivation.cc | 11 ++--------- src/nix/flake.cc | 8 ++++---- src/nix/hash.cc | 11 +++-------- src/nix/main.cc | 2 +- src/nix/nar.cc | 9 +-------- src/nix/profile.cc | 11 +++-------- src/nix/realisation.cc | 11 ++--------- src/nix/registry.cc | 14 ++++---------- src/nix/sigs.cc | 11 +++-------- src/nix/store.cc | 11 ++--------- 15 files changed, 55 insertions(+), 89 deletions(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index de9f546fc..369fa6004 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -1,4 +1,5 @@ #include "command.hh" +#include "markdown.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "derivations.hh" @@ -34,6 +35,19 @@ nlohmann::json NixMultiCommand::toJSON() return MultiCommand::toJSON(); } +void NixMultiCommand::run() +{ + if (!command) { + std::set subCommandTextLines; + for (auto & [name, _] : commands) + subCommandTextLines.insert(fmt("- `%s`", name)); + std::string markdownError = fmt("`nix %s` requires a sub-command. Available sub-commands:\n\n%s\n", + commandName, concatStringsSep("\n", subCommandTextLines)); + throw UsageError(renderMarkdownToTerminal(markdownError)); + } + command->second->run(); +} + StoreCommand::StoreCommand() { } diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 120c832ac..4a72627ed 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -26,9 +26,13 @@ static constexpr Command::Category catNixInstallation = 102; static constexpr auto installablesCategory = "Options that change the interpretation of [installables](@docroot@/command-ref/new-cli/nix.md#installables)"; -struct NixMultiCommand : virtual MultiCommand, virtual Command +struct NixMultiCommand : MultiCommand, virtual Command { nlohmann::json toJSON() override; + + using MultiCommand::MultiCommand; + + virtual void run() override; }; // For the overloaded run methods diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 4480a03f5..c4b2975ee 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -483,7 +483,7 @@ bool Args::processArgs(const Strings & args, bool finish) if (!anyCompleted) exp.handler.fun(ss); - /* Move the list element to the processedArgs. This is almost the same as + /* Move the list element to the processedArgs. This is almost the same as `processedArgs.push_back(expectedArgs.front()); expectedArgs.pop_front()`, except that it will only adjust the next and prev pointers of the list elements, meaning the actual contents don't move in memory. This is @@ -622,8 +622,9 @@ std::optional Command::experimentalFeature () return { Xp::NixCommand }; } -MultiCommand::MultiCommand(const Commands & commands_) +MultiCommand::MultiCommand(std::string_view commandName, const Commands & commands_) : commands(commands_) + , commandName(commandName) { expectArgs({ .label = "subcommand", diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 7af82b178..72278dccc 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -223,11 +223,11 @@ protected: std::list expectedArgs; /** * List of processed positional argument forms. - * + * * All items removed from `expectedArgs` are added here. After all * arguments were processed, this list should be exactly the same as * `expectedArgs` was before. - * + * * This list is used to extend the lifetime of the argument forms. * If this is not done, some closures that reference the command * itself will segfault. @@ -356,13 +356,16 @@ public: */ std::optional>> command; - MultiCommand(const Commands & commands); + MultiCommand(std::string_view commandName, const Commands & commands); bool processFlag(Strings::iterator & pos, Strings::iterator end) override; bool processArgs(const Strings & args, bool finish) override; nlohmann::json toJSON() override; + +protected: + std::string commandName = ""; }; Strings argvToStrings(int argc, char * * argv); diff --git a/src/nix/config.cc b/src/nix/config.cc index 5b280d11d..52706afcf 100644 --- a/src/nix/config.cc +++ b/src/nix/config.cc @@ -7,9 +7,9 @@ using namespace nix; -struct CmdConfig : virtual NixMultiCommand +struct CmdConfig : NixMultiCommand { - CmdConfig() : MultiCommand(RegisterCommand::getCommandsFor({"config"})) + CmdConfig() : NixMultiCommand("config", RegisterCommand::getCommandsFor({"config"})) { } std::string description() override @@ -18,13 +18,6 @@ struct CmdConfig : virtual NixMultiCommand } Category category() override { return catUtility; } - - void run() override - { - if (!command) - throw UsageError("'nix config' requires a sub-command."); - command->second->run(); - } }; struct CmdConfigShow : Command, MixJSON diff --git a/src/nix/derivation.cc b/src/nix/derivation.cc index cd3975a4f..59a78d378 100644 --- a/src/nix/derivation.cc +++ b/src/nix/derivation.cc @@ -2,9 +2,9 @@ using namespace nix; -struct CmdDerivation : virtual NixMultiCommand +struct CmdDerivation : NixMultiCommand { - CmdDerivation() : MultiCommand(RegisterCommand::getCommandsFor({"derivation"})) + CmdDerivation() : NixMultiCommand("derivation", RegisterCommand::getCommandsFor({"derivation"})) { } std::string description() override @@ -13,13 +13,6 @@ struct CmdDerivation : virtual NixMultiCommand } Category category() override { return catUtility; } - - void run() override - { - if (!command) - throw UsageError("'nix derivation' requires a sub-command."); - command->second->run(); - } }; static auto rCmdDerivation = registerCommand("derivation"); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index e0c67fdfa..2b6e56283 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1399,7 +1399,9 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON struct CmdFlake : NixMultiCommand { CmdFlake() - : MultiCommand({ + : NixMultiCommand( + "flake", + { {"update", []() { return make_ref(); }}, {"lock", []() { return make_ref(); }}, {"metadata", []() { return make_ref(); }}, @@ -1429,10 +1431,8 @@ struct CmdFlake : NixMultiCommand void run() override { - if (!command) - throw UsageError("'nix flake' requires a sub-command."); experimentalFeatureSettings.require(Xp::Flakes); - command->second->run(); + NixMultiCommand::run(); } }; diff --git a/src/nix/hash.cc b/src/nix/hash.cc index d6595dcca..ededf6ef2 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -130,7 +130,9 @@ struct CmdToBase : Command struct CmdHash : NixMultiCommand { CmdHash() - : MultiCommand({ + : NixMultiCommand( + "hash", + { {"file", []() { return make_ref(FileIngestionMethod::Flat);; }}, {"path", []() { return make_ref(FileIngestionMethod::Recursive); }}, {"to-base16", []() { return make_ref(HashFormat::Base16); }}, @@ -146,13 +148,6 @@ struct CmdHash : NixMultiCommand } Category category() override { return catUtility; } - - void run() override - { - if (!command) - throw UsageError("'nix hash' requires a sub-command."); - command->second->run(); - } }; static auto rCmdHash = registerCommand("hash"); diff --git a/src/nix/main.cc b/src/nix/main.cc index 3d44e4a9d..109d2cc04 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -67,7 +67,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs bool helpRequested = false; bool showVersion = false; - NixArgs() : MultiCommand(RegisterCommand::getCommandsFor({})), MixCommonArgs("nix") + NixArgs() : MultiCommand("", RegisterCommand::getCommandsFor({})), MixCommonArgs("nix") { categories.clear(); categories[catHelp] = "Help commands"; diff --git a/src/nix/nar.cc b/src/nix/nar.cc index 9815410cf..8ad4f92a7 100644 --- a/src/nix/nar.cc +++ b/src/nix/nar.cc @@ -4,7 +4,7 @@ using namespace nix; struct CmdNar : NixMultiCommand { - CmdNar() : MultiCommand(RegisterCommand::getCommandsFor({"nar"})) + CmdNar() : NixMultiCommand("nar", RegisterCommand::getCommandsFor({"nar"})) { } std::string description() override @@ -20,13 +20,6 @@ struct CmdNar : NixMultiCommand } Category category() override { return catUtility; } - - void run() override - { - if (!command) - throw UsageError("'nix nar' requires a sub-command."); - command->second->run(); - } }; static auto rCmdNar = registerCommand("nar"); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 476ddcd60..147b4680b 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -825,7 +825,9 @@ struct CmdProfileWipeHistory : virtual StoreCommand, MixDefaultProfile, MixDryRu struct CmdProfile : NixMultiCommand { CmdProfile() - : MultiCommand({ + : NixMultiCommand( + "profile", + { {"install", []() { return make_ref(); }}, {"remove", []() { return make_ref(); }}, {"upgrade", []() { return make_ref(); }}, @@ -848,13 +850,6 @@ struct CmdProfile : NixMultiCommand #include "profile.md" ; } - - void run() override - { - if (!command) - throw UsageError("'nix profile' requires a sub-command."); - command->second->run(); - } }; static auto rCmdProfile = registerCommand("profile"); diff --git a/src/nix/realisation.cc b/src/nix/realisation.cc index e19e93219..e1f231222 100644 --- a/src/nix/realisation.cc +++ b/src/nix/realisation.cc @@ -5,9 +5,9 @@ using namespace nix; -struct CmdRealisation : virtual NixMultiCommand +struct CmdRealisation : NixMultiCommand { - CmdRealisation() : MultiCommand(RegisterCommand::getCommandsFor({"realisation"})) + CmdRealisation() : NixMultiCommand("realisation", RegisterCommand::getCommandsFor({"realisation"})) { } std::string description() override @@ -16,13 +16,6 @@ struct CmdRealisation : virtual NixMultiCommand } Category category() override { return catUtility; } - - void run() override - { - if (!command) - throw UsageError("'nix realisation' requires a sub-command."); - command->second->run(); - } }; static auto rCmdRealisation = registerCommand("realisation"); diff --git a/src/nix/registry.cc b/src/nix/registry.cc index f509ccae8..0346ec1e0 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -196,10 +196,12 @@ struct CmdRegistryPin : RegistryCommand, EvalCommand } }; -struct CmdRegistry : virtual NixMultiCommand +struct CmdRegistry : NixMultiCommand { CmdRegistry() - : MultiCommand({ + : NixMultiCommand( + "registry", + { {"list", []() { return make_ref(); }}, {"add", []() { return make_ref(); }}, {"remove", []() { return make_ref(); }}, @@ -221,14 +223,6 @@ struct CmdRegistry : virtual NixMultiCommand } Category category() override { return catSecondary; } - - void run() override - { - experimentalFeatureSettings.require(Xp::Flakes); - if (!command) - throw UsageError("'nix registry' requires a sub-command."); - command->second->run(); - } }; static auto rCmdRegistry = registerCommand("registry"); diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 39555c9ea..a57a407e6 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -205,7 +205,9 @@ struct CmdKeyConvertSecretToPublic : Command struct CmdKey : NixMultiCommand { CmdKey() - : MultiCommand({ + : NixMultiCommand( + "key", + { {"generate-secret", []() { return make_ref(); }}, {"convert-secret-to-public", []() { return make_ref(); }}, }) @@ -218,13 +220,6 @@ struct CmdKey : NixMultiCommand } Category category() override { return catUtility; } - - void run() override - { - if (!command) - throw UsageError("'nix key' requires a sub-command."); - command->second->run(); - } }; static auto rCmdKey = registerCommand("key"); diff --git a/src/nix/store.cc b/src/nix/store.cc index 2879e03b3..79b41e096 100644 --- a/src/nix/store.cc +++ b/src/nix/store.cc @@ -2,9 +2,9 @@ using namespace nix; -struct CmdStore : virtual NixMultiCommand +struct CmdStore : NixMultiCommand { - CmdStore() : MultiCommand(RegisterCommand::getCommandsFor({"store"})) + CmdStore() : NixMultiCommand("store", RegisterCommand::getCommandsFor({"store"})) { } std::string description() override @@ -13,13 +13,6 @@ struct CmdStore : virtual NixMultiCommand } Category category() override { return catUtility; } - - void run() override - { - if (!command) - throw UsageError("'nix store' requires a sub-command."); - command->second->run(); - } }; static auto rCmdStore = registerCommand("store"); From e7abf60a0c8db19927e4fb195789b698c84e8d5a Mon Sep 17 00:00:00 2001 From: Peter Kolloch Date: Sat, 25 Nov 2023 17:33:44 +0100 Subject: [PATCH 077/141] hash.cc/hash.h: Minor C++ improvements --- src/libutil/hash.hh | 14 +++++++------- src/nix/hash.cc | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 6ade6555c..0e5c91b79 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -52,7 +52,7 @@ struct Hash /** * Create a zero-filled hash object. */ - Hash(HashType type); + explicit Hash(HashType type); /** * Parse the hash from a string representation in the format @@ -103,7 +103,7 @@ public: /** * Returns the length of a base-16 representation of this hash. */ - size_t base16Len() const + [[nodiscard]] size_t base16Len() const { return hashSize * 2; } @@ -111,7 +111,7 @@ public: /** * Returns the length of a base-32 representation of this hash. */ - size_t base32Len() const + [[nodiscard]] size_t base32Len() const { return (hashSize * 8 - 1) / 5 + 1; } @@ -119,7 +119,7 @@ public: /** * Returns the length of a base-64 representation of this hash. */ - size_t base64Len() const + [[nodiscard]] size_t base64Len() const { return ((4 * hashSize / 3) + 3) & ~3; } @@ -129,14 +129,14 @@ public: * or base-64. By default, this is prefixed by the hash type * (e.g. "sha256:"). */ - std::string to_string(HashFormat hashFormat, bool includeType) const; + [[nodiscard]] std::string to_string(HashFormat hashFormat, bool includeType) const; - std::string gitRev() const + [[nodiscard]] std::string gitRev() const { return to_string(HashFormat::Base16, false); } - std::string gitShortRev() const + [[nodiscard]] std::string gitShortRev() const { return std::string(to_string(HashFormat::Base16, false), 0, 7); } diff --git a/src/nix/hash.cc b/src/nix/hash.cc index ededf6ef2..cac65006b 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -17,7 +17,7 @@ struct CmdHashBase : Command std::vector paths; std::optional modulus; - CmdHashBase(FileIngestionMethod mode) : mode(mode) + explicit CmdHashBase(FileIngestionMethod mode) : mode(mode) { addFlag({ .longName = "sri", From 156ea78d7402368e3816855800eb6e0ed33a1ecc Mon Sep 17 00:00:00 2001 From: Peter Kolloch Date: Sat, 25 Nov 2023 17:34:16 +0100 Subject: [PATCH 078/141] CmdHashBase: doc comment --- src/nix/hash.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/nix/hash.cc b/src/nix/hash.cc index cac65006b..dfef44221 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -8,6 +8,11 @@ using namespace nix; +/** + * Base for `nix hash file` (deprecated), `nix hash path` and `nix-hash` (legacy). + * + * Deprecation Issue: https://github.com/NixOS/nix/issues/8876 + */ struct CmdHashBase : Command { FileIngestionMethod mode; From 6bbd900d4f9983f74dcd9a0f85ab899331f661c7 Mon Sep 17 00:00:00 2001 From: Peter Kolloch Date: Sat, 25 Nov 2023 17:35:24 +0100 Subject: [PATCH 079/141] nix hash convert: added This deviated from the proposal! See comments on the issue. https://github.com/NixOS/nix/issues/8876 --- src/nix/hash.cc | 63 ++++++++++++++++++++++++++++++++++++++++ tests/functional/hash.sh | 19 +++++++++++- 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/src/nix/hash.cc b/src/nix/hash.cc index dfef44221..2b32ac03c 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -132,12 +132,75 @@ struct CmdToBase : Command } }; +/** + * `nix hash convert` + */ +struct CmdHashConvert : Command +{ + std::optional from; + HashFormat to; + std::optional type; + std::vector hashStrings; + + CmdHashConvert(): to(HashFormat::SRI) { + addFlag({ + .longName = "from", + // TODO: List format choices. Maybe introduce a constant? + .description = "The format of the input hash.", + .labels = {"hash format"}, + .handler = {[this](std::string str) { + from = parseHashFormat(str); + }}, + }); + addFlag({ + .longName = "to", + // TODO: List format choices. Maybe introduce a constant? + .description = "The format of the output hash.", + .labels = {"hash format"}, + .handler = {[this](std::string str) { + to = parseHashFormat(str); + }}, + }); + addFlag({ + .longName = "type", + .description = "Specify the type if it can't be auto-detected.", + .labels = {"hash type"}, + .handler = {[this](std::string str) { + type = parseHashType(str); + }}, + }); + expectArgs({ + .label = "hashes", + .handler = {&hashStrings}, + }); + } + + std::string description() override + { + return "convert between different hash formats, e.g. base16 and sri."; + } + + Category category() override { return catUtility; } + + void run() override { + for (const auto& s: hashStrings) { + Hash h = Hash::parseAny(s, type); + if (from && h.to_string(*from, from == HashFormat::SRI) != s) { + auto from_as_string = printHashFormat(*from); + throw BadHash("input hash '%s' does not have the expected format '--from %s'", s, from_as_string); + } + logger->cout(h.to_string(to, to == HashFormat::SRI)); + } + } +}; + struct CmdHash : NixMultiCommand { CmdHash() : NixMultiCommand( "hash", { + {"convert", []() { return make_ref();}}, {"file", []() { return make_ref(FileIngestionMethod::Flat);; }}, {"path", []() { return make_ref(FileIngestionMethod::Recursive); }}, {"to-base16", []() { return make_ref(HashFormat::Base16); }}, diff --git a/tests/functional/hash.sh b/tests/functional/hash.sh index 34c1bb38a..d66b27a26 100644 --- a/tests/functional/hash.sh +++ b/tests/functional/hash.sh @@ -81,24 +81,41 @@ rm $TEST_ROOT/hash-path/hello ln -s x $TEST_ROOT/hash-path/hello try2 md5 "f78b733a68f5edbdf9413899339eaa4a" -# Conversion. +# Conversion with `nix hash` `nix-hash` and `nix hash convert` try3() { + # $1 = hash type + # $2 = expected hash in base16 + # $3 = expected hash in base32 + # $4 = expected hash in base64 + h64=$(nix hash convert --type "$1" --to base64 "$2") + [ "$h64" = "$4" ] h64=$(nix-hash --type "$1" --to-base64 "$2") [ "$h64" = "$4" ] + # Deprecated experiment h64=$(nix hash to-base64 --type "$1" "$2") [ "$h64" = "$4" ] + + sri=$(nix hash convert --type "$1" --to sri "$2") + [ "$sri" = "$1-$4" ] sri=$(nix-hash --type "$1" --to-sri "$2") [ "$sri" = "$1-$4" ] sri=$(nix hash to-sri --type "$1" "$2") [ "$sri" = "$1-$4" ] + h32=$(nix hash convert --type "$1" --to base32 "$2") + [ "$h32" = "$3" ] h32=$(nix-hash --type "$1" --to-base32 "$2") [ "$h32" = "$3" ] h32=$(nix hash to-base32 --type "$1" "$2") [ "$h32" = "$3" ] h16=$(nix-hash --type "$1" --to-base16 "$h32") [ "$h16" = "$2" ] + + h16=$(nix hash convert --type "$1" --to base16 "$h64") + [ "$h16" = "$2" ] h16=$(nix hash to-base16 --type "$1" "$h64") [ "$h16" = "$2" ] + h16=$(nix hash convert --to base16 "$sri") + [ "$h16" = "$2" ] h16=$(nix hash to-base16 "$sri") [ "$h16" = "$2" ] } From 0c2d5f7673ae0196b660c39b59941755103c23d0 Mon Sep 17 00:00:00 2001 From: Peter Kolloch Date: Tue, 28 Nov 2023 11:42:52 +0100 Subject: [PATCH 080/141] nix hash convert: s/--type/--algo/ + more functional tests https://github.com/NixOS/nix/issues/8876 --- src/libutil/hash.hh | 1 - src/nix/hash.cc | 8 ++--- tests/functional/hash.sh | 72 +++++++++++++++++++++++++++++++++++++--- 3 files changed, 71 insertions(+), 10 deletions(-) diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 0e5c91b79..820154e7a 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -40,7 +40,6 @@ enum struct HashFormat : int { SRI }; - struct Hash { constexpr static size_t maxHashSize = 64; diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 2b32ac03c..62f96ef1d 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -162,9 +162,9 @@ struct CmdHashConvert : Command }}, }); addFlag({ - .longName = "type", - .description = "Specify the type if it can't be auto-detected.", - .labels = {"hash type"}, + .longName = "algo", + .description = "Specify the algorithm if it can't be auto-detected.", + .labels = {"hash algorithm"}, .handler = {[this](std::string str) { type = parseHashType(str); }}, @@ -177,7 +177,7 @@ struct CmdHashConvert : Command std::string description() override { - return "convert between different hash formats, e.g. base16 and sri."; + return "convert between different hash formats, e.g. base16, nix32, base64 and sri."; } Category category() override { return catUtility; } diff --git a/tests/functional/hash.sh b/tests/functional/hash.sh index d66b27a26..031e33adf 100644 --- a/tests/functional/hash.sh +++ b/tests/functional/hash.sh @@ -83,11 +83,11 @@ try2 md5 "f78b733a68f5edbdf9413899339eaa4a" # Conversion with `nix hash` `nix-hash` and `nix hash convert` try3() { - # $1 = hash type + # $1 = hash algo # $2 = expected hash in base16 # $3 = expected hash in base32 # $4 = expected hash in base64 - h64=$(nix hash convert --type "$1" --to base64 "$2") + h64=$(nix hash convert --algo "$1" --to base64 "$2") [ "$h64" = "$4" ] h64=$(nix-hash --type "$1" --to-base64 "$2") [ "$h64" = "$4" ] @@ -95,13 +95,13 @@ try3() { h64=$(nix hash to-base64 --type "$1" "$2") [ "$h64" = "$4" ] - sri=$(nix hash convert --type "$1" --to sri "$2") + sri=$(nix hash convert --algo "$1" --to sri "$2") [ "$sri" = "$1-$4" ] sri=$(nix-hash --type "$1" --to-sri "$2") [ "$sri" = "$1-$4" ] sri=$(nix hash to-sri --type "$1" "$2") [ "$sri" = "$1-$4" ] - h32=$(nix hash convert --type "$1" --to base32 "$2") + h32=$(nix hash convert --algo "$1" --to base32 "$2") [ "$h32" = "$3" ] h32=$(nix-hash --type "$1" --to-base32 "$2") [ "$h32" = "$3" ] @@ -110,7 +110,7 @@ try3() { h16=$(nix-hash --type "$1" --to-base16 "$h32") [ "$h16" = "$2" ] - h16=$(nix hash convert --type "$1" --to base16 "$h64") + h16=$(nix hash convert --algo "$1" --to base16 "$h64") [ "$h16" = "$2" ] h16=$(nix hash to-base16 --type "$1" "$h64") [ "$h16" = "$2" ] @@ -118,7 +118,69 @@ try3() { [ "$h16" = "$2" ] h16=$(nix hash to-base16 "$sri") [ "$h16" = "$2" ] + + # + # Converting from SRI + # + + # Input hash algo auto-detected from SRI and output defaults to SRI as well. + sri=$(nix hash convert "$1-$4") + [ "$sri" = "$1-$4" ] + + sri=$(nix hash convert --from sri "$1-$4") + [ "$sri" = "$1-$4" ] + + sri=$(nix hash convert --to sri "$1-$4") + [ "$sri" = "$1-$4" ] + + sri=$(nix hash convert --from sri --to sri "$1-$4") + [ "$sri" = "$1-$4" ] + + sri=$(nix hash convert --to base64 "$1-$4") + [ "$sri" = "$4" ] + + # + # Auto-detecting the input from algo and length. + # + + sri=$(nix hash convert --algo "$1" "$2") + [ "$sri" = "$1-$4" ] + sri=$(nix hash convert --algo "$1" "$3") + [ "$sri" = "$1-$4" ] + sri=$(nix hash convert --algo "$1" "$4") + [ "$sri" = "$1-$4" ] + + sri=$(nix hash convert --algo "$1" "$2") + [ "$sri" = "$1-$4" ] + sri=$(nix hash convert --algo "$1" "$3") + [ "$sri" = "$1-$4" ] + sri=$(nix hash convert --algo "$1" "$4") + [ "$sri" = "$1-$4" ] + + # + # Asserting input format succeeds. + # + + sri=$(nix hash convert --algo "$1" --from base16 "$2") + [ "$sri" = "$1-$4" ] + sri=$(nix hash convert --algo "$1" --from base32 "$3") + [ "$sri" = "$1-$4" ] + sri=$(nix hash convert --algo "$1" --from base64 "$4") + [ "$sri" = "$1-$4" ] + + # + # Asserting input format fails. + # + + fail=$(nix hash convert --algo "$1" --from base32 "$2" 2>&1 || echo "exit: $?") + [[ "$fail" == "error: input hash"*"exit: 1" ]] + fail=$(nix hash convert --algo "$1" --from base16 "$3" 2>&1 || echo "exit: $?") + [[ "$fail" == "error: input hash"*"exit: 1" ]] + fail=$(nix hash convert --algo "$1" --from base32 "$4" 2>&1 || echo "exit: $?") + [[ "$fail" == "error: input hash"*"exit: 1" ]] + } + try3 sha1 "800d59cfcd3c05e900cb4e214be48f6b886a08df" "vw46m23bizj4n8afrc0fj19wrp7mj3c0" "gA1Zz808BekAy04hS+SPa4hqCN8=" try3 sha256 "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=" try3 sha512 "204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445" "12k9jiq29iyqm03swfsgiw5mlqs173qazm3n7daz43infy12pyrcdf30fkk3qwv4yl2ick8yipc2mqnlh48xsvvxl60lbx8vp38yji0" "IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ==" From 5334c9c792a208db4d3824e88019a626ded1b65d Mon Sep 17 00:00:00 2001 From: Peter Kolloch Date: Tue, 28 Nov 2023 14:20:27 +0100 Subject: [PATCH 081/141] HashType: Rename to HashAlgorithm To be consistent with CLI, nix API and many other references. As part of this, we also converted it to a scoped enum. https://github.com/NixOS/nix/issues/8876 --- perl/lib/Nix/Store.xs | 12 +- src/libexpr/flake/flake.cc | 2 +- src/libexpr/primops.cc | 36 ++--- src/libexpr/primops/fetchMercurial.cc | 4 +- src/libexpr/primops/fetchTree.cc | 8 +- src/libfetchers/fetchers.cc | 6 +- src/libfetchers/git-utils.cc | 4 +- src/libfetchers/git.cc | 6 +- src/libfetchers/github.cc | 10 +- src/libfetchers/indirect.cc | 4 +- src/libfetchers/input-accessor.cc | 4 +- src/libfetchers/mercurial.cc | 10 +- src/libfetchers/tarball.cc | 6 +- src/libstore/binary-cache-store.cc | 24 +-- src/libstore/binary-cache-store.hh | 16 +- src/libstore/build/local-derivation-goal.cc | 42 +++--- src/libstore/build/worker.cc | 4 +- src/libstore/builtins/fetchurl.cc | 4 +- src/libstore/content-address.cc | 28 ++-- src/libstore/content-address.hh | 4 +- src/libstore/daemon.cc | 16 +- src/libstore/derivations.cc | 64 ++++---- src/libstore/derivations.hh | 8 +- src/libstore/downstream-placeholder.cc | 4 +- src/libstore/export-import.cc | 6 +- src/libstore/gc.cc | 2 +- src/libstore/legacy-ssh-store.cc | 14 +- src/libstore/local-store.cc | 30 ++-- src/libstore/local-store.hh | 6 +- src/libstore/make-content-addressed.cc | 4 +- src/libstore/nar-info.cc | 4 +- src/libstore/optimise-store.cc | 4 +- src/libstore/path-references.cc | 2 +- src/libstore/path.cc | 2 +- src/libstore/remote-store.cc | 28 ++-- src/libstore/remote-store.hh | 14 +- src/libstore/store-api.cc | 46 +++--- src/libstore/store-api.hh | 22 +-- src/libstore/store-dir-config.hh | 2 +- src/libstore/worker-protocol.cc | 2 +- src/libutil/args.cc | 14 +- src/libutil/args.hh | 6 +- src/libutil/git.cc | 8 +- src/libutil/git.hh | 6 +- src/libutil/hash.cc | 158 ++++++++++---------- src/libutil/hash.hh | 38 ++--- src/libutil/references.cc | 4 +- src/libutil/references.hh | 2 +- src/libutil/source-accessor.cc | 8 +- src/libutil/source-accessor.hh | 6 +- src/nix-store/nix-store.cc | 12 +- src/nix/add-to-store.cc | 4 +- src/nix/hash.cc | 24 +-- src/nix/prefetch.cc | 34 ++--- src/nix/profile.cc | 2 +- src/nix/verify.cc | 2 +- tests/unit/libstore/common-protocol.cc | 8 +- tests/unit/libstore/derivation.cc | 4 +- tests/unit/libstore/nar-info.cc | 2 +- tests/unit/libstore/path-info.cc | 2 +- tests/unit/libstore/serve-protocol.cc | 8 +- tests/unit/libstore/worker-protocol.cc | 10 +- tests/unit/libutil/git.cc | 8 +- tests/unit/libutil/hash.cc | 16 +- 64 files changed, 450 insertions(+), 450 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 40257ed74..50148141b 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -205,7 +205,7 @@ void importPaths(int fd, int dontCheckSigs) SV * hashPath(char * algo, int base32, char * path) PPCODE: try { - Hash h = hashPath(parseHashType(algo), path).first; + Hash h = hashPath(parseHashAlgo(algo), path).first; auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { @@ -216,7 +216,7 @@ SV * hashPath(char * algo, int base32, char * path) SV * hashFile(char * algo, int base32, char * path) PPCODE: try { - Hash h = hashFile(parseHashType(algo), path); + Hash h = hashFile(parseHashAlgo(algo), path); auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { @@ -227,7 +227,7 @@ SV * hashFile(char * algo, int base32, char * path) SV * hashString(char * algo, int base32, char * s) PPCODE: try { - Hash h = hashString(parseHashType(algo), s); + Hash h = hashString(parseHashAlgo(algo), s); auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { @@ -238,7 +238,7 @@ SV * hashString(char * algo, int base32, char * s) SV * convertHash(char * algo, char * s, int toBase32) PPCODE: try { - auto h = Hash::parseAny(s, parseHashType(algo)); + auto h = Hash::parseAny(s, parseHashAlgo(algo)); auto s = h.to_string(toBase32 ? HashFormat::Base32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { @@ -281,7 +281,7 @@ SV * addToStore(char * srcPath, int recursive, char * algo) PPCODE: try { auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, method, parseHashType(algo)); + auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, method, parseHashAlgo(algo)); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -291,7 +291,7 @@ SV * addToStore(char * srcPath, int recursive, char * algo) SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) PPCODE: try { - auto h = Hash::parseAny(hash, parseHashType(algo)); + auto h = Hash::parseAny(hash, parseHashAlgo(algo)); auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; auto path = store()->makeFixedOutputPath(name, FixedOutputInfo { .method = method, diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 8f8fc64f0..fee58792b 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -904,7 +904,7 @@ Fingerprint LockedFlake::getFingerprint() const // FIXME: as an optimization, if the flake contains a lock file // and we haven't changed it, then it's sufficient to use // flake.sourceInfo.storePath for the fingerprint. - return hashString(htSHA256, + return hashString(HashAlgorithm::SHA256, fmt("%s;%s;%d;%d;%s", flake.storePath.to_string(), flake.lockedRef.subdir, diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index c2499bdae..7831f3803 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1317,7 +1317,7 @@ drvName, Bindings * attrs, Value & v) .errPos = state.positions[noPos] })); - auto h = newHashAllowEmpty(*outputHash, parseHashTypeOpt(outputHashAlgo)); + auto h = newHashAllowEmpty(*outputHash, parseHashAlgoOpt(outputHashAlgo)); auto method = ingestionMethod.value_or(FileIngestionMethod::Flat); @@ -1339,7 +1339,7 @@ drvName, Bindings * attrs, Value & v) .errPos = state.positions[noPos] }); - auto ht = parseHashTypeOpt(outputHashAlgo).value_or(htSHA256); + auto ht = parseHashAlgoOpt(outputHashAlgo).value_or(HashAlgorithm::SHA256); auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive); for (auto & i : outputs) { @@ -1348,13 +1348,13 @@ drvName, Bindings * attrs, Value & v) drv.outputs.insert_or_assign(i, DerivationOutput::Impure { .method = method, - .hashType = ht, + .hashAlgo = ht, }); else drv.outputs.insert_or_assign(i, DerivationOutput::CAFloating { .method = method, - .hashType = ht, + .hashAlgo = ht, }); } } @@ -1754,17 +1754,17 @@ static RegisterPrimOp primop_findFile(PrimOp { /* Return the cryptographic hash of a file in base-16. */ static void prim_hashFile(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto type = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashFile"); - std::optional ht = parseHashType(type); - if (!ht) + auto algo = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashFile"); + std::optional ha = parseHashAlgo(algo); + if (!ha) state.debugThrowLastTrace(Error({ - .msg = hintfmt("unknown hash type '%1%'", type), + .msg = hintfmt("unknown hash algo '%1%'", algo), .errPos = state.positions[pos] })); auto path = realisePath(state, pos, *args[1]); - v.mkString(hashString(*ht, path.readFile()).to_string(HashFormat::Base16, false)); + v.mkString(hashString(*ha, path.readFile()).to_string(HashFormat::Base16, false)); } static RegisterPrimOp primop_hashFile({ @@ -2341,7 +2341,7 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value else if (n == "recursive") method = FileIngestionMethod { state.forceBool(*attr.value, attr.pos, "while evaluating the `recursive` attribute passed to builtins.path") }; else if (n == "sha256") - expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `sha256` attribute passed to builtins.path"), htSHA256); + expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `sha256` attribute passed to builtins.path"), HashAlgorithm::SHA256); else state.debugThrowLastTrace(EvalError({ .msg = hintfmt("unsupported argument '%1%' to 'addPath'", state.symbols[attr.name]), @@ -3766,18 +3766,18 @@ static RegisterPrimOp primop_stringLength({ /* Return the cryptographic hash of a string in base-16. */ static void prim_hashString(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto type = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashString"); - std::optional ht = parseHashType(type); - if (!ht) + auto algo = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashString"); + std::optional ha = parseHashAlgo(algo); + if (!ha) state.debugThrowLastTrace(Error({ - .msg = hintfmt("unknown hash type '%1%'", type), + .msg = hintfmt("unknown hash algo '%1%'", algo), .errPos = state.positions[pos] })); NixStringContext context; // discarded auto s = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString"); - v.mkString(hashString(*ht, s).to_string(HashFormat::Base16, false)); + v.mkString(hashString(*ha, s).to_string(HashFormat::Base16, false)); } static RegisterPrimOp primop_hashString({ @@ -3800,15 +3800,15 @@ static void prim_convertHash(EvalState & state, const PosIdx pos, Value * * args auto hash = state.forceStringNoCtx(*iteratorHash->value, pos, "while evaluating the attribute 'hash'"); Bindings::iterator iteratorHashAlgo = inputAttrs->find(state.symbols.create("hashAlgo")); - std::optional ht = std::nullopt; + std::optional ha = std::nullopt; if (iteratorHashAlgo != inputAttrs->end()) { - ht = parseHashType(state.forceStringNoCtx(*iteratorHashAlgo->value, pos, "while evaluating the attribute 'hashAlgo'")); + ha = parseHashAlgo(state.forceStringNoCtx(*iteratorHashAlgo->value, pos, "while evaluating the attribute 'hashAlgo'")); } Bindings::iterator iteratorToHashFormat = getAttr(state, state.symbols.create("toHashFormat"), args[0]->attrs, "while locating the attribute 'toHashFormat'"); HashFormat hf = parseHashFormat(state.forceStringNoCtx(*iteratorToHashFormat->value, pos, "while evaluating the attribute 'toHashFormat'")); - v.mkString(Hash::parseAny(hash, ht).to_string(hf, hf == HashFormat::SRI)); + v.mkString(Hash::parseAny(hash, ha).to_string(hf, hf == HashFormat::SRI)); } static RegisterPrimOp primop_convertHash({ diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index e76ce455d..58fe6f173 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -31,7 +31,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a // be both a revision or a branch/tag name. auto value = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `rev` attribute passed to builtins.fetchMercurial"); if (std::regex_match(value.begin(), value.end(), revRegex)) - rev = Hash::parseAny(value, htSHA1); + rev = Hash::parseAny(value, HashAlgorithm::SHA1); else ref = value; } @@ -79,7 +79,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a attrs2.alloc("branch").mkString(*input2.getRef()); // Backward compatibility: set 'rev' to // 0000000000000000000000000000000000000000 for a dirty tree. - auto rev2 = input2.getRev().value_or(Hash(htSHA1)); + auto rev2 = input2.getRev().value_or(Hash(HashAlgorithm::SHA1)); attrs2.alloc("rev").mkString(rev2.gitRev()); attrs2.alloc("shortRev").mkString(rev2.gitRev().substr(0, 12)); if (auto revCount = input2.getRevCount()) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 383ec7c58..ef80c634f 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -46,7 +46,7 @@ void emitTreeAttrs( attrs.alloc("shortRev").mkString(rev->gitShortRev()); } else if (emptyRevFallback) { // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev - auto emptyHash = Hash(htSHA1); + auto emptyHash = Hash(HashAlgorithm::SHA1); attrs.alloc("rev").mkString(emptyHash.gitRev()); attrs.alloc("shortRev").mkString(emptyHash.gitShortRev()); } @@ -246,7 +246,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v if (n == "url") url = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the url we should fetch"); else if (n == "sha256") - expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the sha256 of the content we should fetch"), htSHA256); + expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the sha256 of the content we should fetch"), HashAlgorithm::SHA256); else if (n == "name") name = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the name of the content we should fetch"); else @@ -276,7 +276,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v state.debugThrowLastTrace(EvalError("in pure evaluation mode, '%s' requires a 'sha256' argument", who)); // early exit if pinned and already in the store - if (expectedHash && expectedHash->type == htSHA256) { + if (expectedHash && expectedHash->algo == HashAlgorithm::SHA256) { auto expectedPath = state.store->makeFixedOutputPath( name, FixedOutputInfo { @@ -301,7 +301,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v if (expectedHash) { auto hash = unpack ? state.store->queryPathInfo(storePath)->narHash - : hashFile(htSHA256, state.store->toRealPath(storePath)); + : hashFile(HashAlgorithm::SHA256, state.store->toRealPath(storePath)); if (hash != *expectedHash) state.debugThrowLastTrace(EvalError((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s", *url, expectedHash->to_string(HashFormat::Base32, true), hash.to_string(HashFormat::Base32, true))); diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 60208619e..573341a3d 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -289,8 +289,8 @@ std::string Input::getType() const std::optional Input::getNarHash() const { if (auto s = maybeGetStrAttr(attrs, "narHash")) { - auto hash = s->empty() ? Hash(htSHA256) : Hash::parseSRI(*s); - if (hash.type != htSHA256) + auto hash = s->empty() ? Hash(HashAlgorithm::SHA256) : Hash::parseSRI(*s); + if (hash.algo != HashAlgorithm::SHA256) throw UsageError("narHash must use SHA-256"); return hash; } @@ -314,7 +314,7 @@ std::optional Input::getRev() const } catch (BadHash &e) { // Default to sha1 for backwards compatibility with existing // usages (e.g. `builtins.fetchTree` calls or flake inputs). - hash = Hash::parseAny(*s, htSHA1); + hash = Hash::parseAny(*s, HashAlgorithm::SHA1); } } diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 19eae0e1d..9356e5817 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -91,7 +91,7 @@ Hash toHash(const git_oid & oid) #ifdef GIT_EXPERIMENTAL_SHA256 assert(oid.type == GIT_OID_SHA1); #endif - Hash hash(htSHA1); + Hash hash(HashAlgorithm::SHA1); memcpy(hash.hash, oid.id, hash.hashSize); return hash; } @@ -439,7 +439,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this std::string re = R"(Good "git" signature for \* with .* key SHA256:[)"; for (const fetchers::PublicKey & k : publicKeys){ // Calculate sha256 fingerprint from public key and escape the regex symbol '+' to match the key literally - auto fingerprint = trim(hashString(htSHA256, base64Decode(k.key)).to_string(nix::HashFormat::Base64, false), "="); + auto fingerprint = trim(hashString(HashAlgorithm::SHA256, base64Decode(k.key)).to_string(nix::HashFormat::Base64, false), "="); auto escaped_fingerprint = std::regex_replace(fingerprint, std::regex("\\+"), "\\+" ); re += "(" + escaped_fingerprint + ")"; } diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 8cd74057c..a89acc1c0 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -52,7 +52,7 @@ bool touchCacheFile(const Path & path, time_t touch_time) Path getCachePath(std::string_view key) { return getCacheDir() + "/nix/gitv3/" + - hashString(htSHA256, key).to_string(HashFormat::Base32, false); + hashString(HashAlgorithm::SHA256, key).to_string(HashFormat::Base32, false); } // Returns the name of the HEAD branch. @@ -369,7 +369,7 @@ struct GitInputScheme : InputScheme { auto checkHashType = [&](const std::optional & hash) { - if (hash.has_value() && !(hash->type == htSHA1 || hash->type == htSHA256)) + if (hash.has_value() && !(hash->algo == HashAlgorithm::SHA1 || hash->algo == HashAlgorithm::SHA256)) throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(HashFormat::Base16, true)); }; @@ -559,7 +559,7 @@ struct GitInputScheme : InputScheme repoInfo.url ); } else - input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), htSHA1).gitRev()); + input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), HashAlgorithm::SHA1).gitRev()); // cache dir lock is removed at scope end; we will only use read-only operations on specific revisions in the remainder } diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 661ad4884..70acb9354 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -42,7 +42,7 @@ struct GitArchiveInputScheme : InputScheme auto size = path.size(); if (size == 3) { if (std::regex_match(path[2], revRegex)) - rev = Hash::parseAny(path[2], htSHA1); + rev = Hash::parseAny(path[2], HashAlgorithm::SHA1); else if (std::regex_match(path[2], refRegex)) ref = path[2]; else @@ -68,7 +68,7 @@ struct GitArchiveInputScheme : InputScheme if (name == "rev") { if (rev) throw BadURL("URL '%s' contains multiple commit hashes", url.url); - rev = Hash::parseAny(value, htSHA1); + rev = Hash::parseAny(value, HashAlgorithm::SHA1); } else if (name == "ref") { if (!std::regex_match(value, refRegex)) @@ -284,7 +284,7 @@ struct GitHubInputScheme : GitArchiveInputScheme readFile( store->toRealPath( downloadFile(store, url, "source", false, headers).storePath))); - auto rev = Hash::parseAny(std::string { json["sha"] }, htSHA1); + auto rev = Hash::parseAny(std::string { json["sha"] }, HashAlgorithm::SHA1); debug("HEAD revision for '%s' is %s", url, rev.gitRev()); return rev; } @@ -356,7 +356,7 @@ struct GitLabInputScheme : GitArchiveInputScheme readFile( store->toRealPath( downloadFile(store, url, "source", false, headers).storePath))); - auto rev = Hash::parseAny(std::string(json[0]["id"]), htSHA1); + auto rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1); debug("HEAD revision for '%s' is %s", url, rev.gitRev()); return rev; } @@ -448,7 +448,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme if(!id) throw BadURL("in '%d', couldn't find ref '%d'", input.to_string(), ref); - auto rev = Hash::parseAny(*id, htSHA1); + auto rev = Hash::parseAny(*id, HashAlgorithm::SHA1); debug("HEAD revision for '%s' is %s", fmt("%s/%s", base_url, ref), rev.gitRev()); return rev; } diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index 8e30284c6..002c0c292 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -20,7 +20,7 @@ struct IndirectInputScheme : InputScheme if (path.size() == 1) { } else if (path.size() == 2) { if (std::regex_match(path[1], revRegex)) - rev = Hash::parseAny(path[1], htSHA1); + rev = Hash::parseAny(path[1], HashAlgorithm::SHA1); else if (std::regex_match(path[1], refRegex)) ref = path[1]; else @@ -31,7 +31,7 @@ struct IndirectInputScheme : InputScheme ref = path[1]; if (!std::regex_match(path[2], revRegex)) throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]); - rev = Hash::parseAny(path[2], htSHA1); + rev = Hash::parseAny(path[2], HashAlgorithm::SHA1); } else throw BadURL("GitHub URL '%s' is invalid", url.url); diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc index 85dc4609f..eabef55d8 100644 --- a/src/libfetchers/input-accessor.cc +++ b/src/libfetchers/input-accessor.cc @@ -44,8 +44,8 @@ StorePath InputAccessor::fetchToStore( auto storePath = settings.readOnlyMode - ? store->computeStorePathFromDump(*source, name, method, htSHA256).first - : store->addToStoreFromDump(*source, name, method, htSHA256, repair); + ? store->computeStorePathFromDump(*source, name, method, HashAlgorithm::SHA256).first + : store->addToStoreFromDump(*source, name, method, HashAlgorithm::SHA256, repair); if (cacheKey) fetchers::getCache()->add(store, *cacheKey, {}, storePath, true); diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index aa991a75d..713f24bbb 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -210,7 +210,7 @@ struct MercurialInputScheme : InputScheme return files.count(file); }; - auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter); + auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, HashAlgorithm::SHA256, filter); return {std::move(storePath), input}; } @@ -220,7 +220,7 @@ struct MercurialInputScheme : InputScheme auto checkHashType = [&](const std::optional & hash) { - if (hash.has_value() && hash->type != htSHA1) + if (hash.has_value() && hash->algo != HashAlgorithm::SHA1) throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", hash->to_string(HashFormat::Base16, true)); }; @@ -260,14 +260,14 @@ struct MercurialInputScheme : InputScheme }); if (auto res = getCache()->lookup(store, unlockedAttrs)) { - auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1); + auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), HashAlgorithm::SHA1); if (!input.getRev() || input.getRev() == rev2) { input.attrs.insert_or_assign("rev", rev2.gitRev()); return makeResult(res->first, std::move(res->second)); } } - Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(HashFormat::Base32, false)); + Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(HashAlgorithm::SHA256, actualUrl).to_string(HashFormat::Base32, false)); /* If this is a commit hash that we already have, we don't have to pull again. */ @@ -301,7 +301,7 @@ struct MercurialInputScheme : InputScheme runHg({ "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" })); assert(tokens.size() == 3); - input.attrs.insert_or_assign("rev", Hash::parseAny(tokens[0], htSHA1).gitRev()); + input.attrs.insert_or_assign("rev", Hash::parseAny(tokens[0], HashAlgorithm::SHA1).gitRev()); auto revCount = std::stoull(tokens[1]); input.attrs.insert_or_assign("ref", tokens[2]); diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 0062878a9..086366180 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -73,7 +73,7 @@ DownloadFileResult downloadFile( } else { StringSink sink; dumpString(res.data, sink); - auto hash = hashString(htSHA256, res.data); + auto hash = hashString(HashAlgorithm::SHA256, res.data); ValidPathInfo info { *store, name, @@ -82,7 +82,7 @@ DownloadFileResult downloadFile( .hash = hash, .references = {}, }, - hashString(htSHA256, sink.s), + hashString(HashAlgorithm::SHA256, sink.s), }; info.narSize = sink.s.size(); auto source = StringSource { sink.s }; @@ -156,7 +156,7 @@ DownloadTarballResult downloadTarball( throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); auto topDir = tmpDir + "/" + members.begin()->name; lastModified = lstat(topDir).st_mtime; - unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, NoRepair); + unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, HashAlgorithm::SHA256, defaultPathFilter, NoRepair); } Attrs infoAttrs({ diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index ae483c95e..f287d72a8 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -143,9 +143,9 @@ ref BinaryCacheStore::addToStoreCommon( /* Read the NAR simultaneously into a CompressionSink+FileSink (to write the compressed NAR to disk), into a HashSink (to get the NAR hash), and into a NarAccessor (to get the NAR listing). */ - HashSink fileHashSink { htSHA256 }; + HashSink fileHashSink { HashAlgorithm::SHA256 }; std::shared_ptr narAccessor; - HashSink narHashSink { htSHA256 }; + HashSink narHashSink { HashAlgorithm::SHA256 }; { FdSink fileSink(fdTemp.get()); TeeSink teeSinkCompressed { fileSink, fileHashSink }; @@ -301,9 +301,9 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource } StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) + FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) { - if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) + if (method != FileIngestionMethod::Recursive || hashAlgo != HashAlgorithm::SHA256) unsupported("addToStoreFromDump"); return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { @@ -399,13 +399,13 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, } StorePath BinaryCacheStore::addToStore( - std::string_view name, - const Path & srcPath, - FileIngestionMethod method, - HashType hashAlgo, - PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) + std::string_view name, + const Path & srcPath, + FileIngestionMethod method, + HashAlgorithm hashAlgo, + PathFilter & filter, + RepairFlag repair, + const StorePathSet & references) { /* FIXME: Make BinaryCacheStore::addToStoreCommon support non-recursive+sha256 so we can just use the default @@ -448,7 +448,7 @@ StorePath BinaryCacheStore::addTextToStore( const StorePathSet & references, RepairFlag repair) { - auto textHash = hashString(htSHA256, s); + auto textHash = hashString(HashAlgorithm::SHA256, s); auto path = makeTextPath(name, TextInfo { { textHash }, references }); if (!repair && isValidPath(path)) diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index cea2a571f..395e1b479 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -124,16 +124,16 @@ public: RepairFlag repair, CheckSigsFlag checkSigs) override; StorePath addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override; + FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) override; StorePath addToStore( - std::string_view name, - const Path & srcPath, - FileIngestionMethod method, - HashType hashAlgo, - PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) override; + std::string_view name, + const Path & srcPath, + FileIngestionMethod method, + HashAlgorithm hashAlgo, + PathFilter & filter, + RepairFlag repair, + const StorePathSet & references) override; StorePath addTextToStore( std::string_view name, diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 198402ff7..4c3dc1f5c 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1066,7 +1066,7 @@ void LocalDerivationGoal::initTmpDir() { if (passAsFile.find(i.first) == passAsFile.end()) { env[i.first] = i.second; } else { - auto hash = hashString(htSHA256, i.first); + auto hash = hashString(HashAlgorithm::SHA256, i.first); std::string fn = ".attr-" + hash.to_string(HashFormat::Base32, false); Path p = tmpDir + "/" + fn; writeFile(p, rewriteStrings(i.second, inputRewrites)); @@ -1290,13 +1290,13 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In { throw Error("queryPathFromHashPart"); } StorePath addToStore( - std::string_view name, - const Path & srcPath, - FileIngestionMethod method, - HashType hashAlgo, - PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) override + std::string_view name, + const Path & srcPath, + FileIngestionMethod method, + HashAlgorithm hashAlgo, + PathFilter & filter, + RepairFlag repair, + const StorePathSet & references) override { throw Error("addToStore"); } void addToStore(const ValidPathInfo & info, Source & narSource, @@ -1318,12 +1318,12 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In } StorePath addToStoreFromDump( - Source & dump, - std::string_view name, - FileIngestionMethod method, - HashType hashAlgo, - RepairFlag repair, - const StorePathSet & references) override + Source & dump, + std::string_view name, + FileIngestionMethod method, + HashAlgorithm hashAlgo, + RepairFlag repair, + const StorePathSet & references) override { auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair, references); goal.addDependency(path); @@ -2466,7 +2466,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() rewriteOutput(outputRewrites); /* FIXME optimize and deduplicate with addToStore */ std::string oldHashPart { scratchPath->hashPart() }; - HashModuloSink caSink { outputHash.hashType, oldHashPart }; + HashModuloSink caSink {outputHash.hashAlgo, oldHashPart }; std::visit(overloaded { [&](const TextIngestionMethod &) { readFile(actualPath, caSink); @@ -2511,7 +2511,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() std::string(newInfo0.path.hashPart())}}); } - HashResult narHashAndSize = hashPath(htSHA256, actualPath); + HashResult narHashAndSize = hashPath(HashAlgorithm::SHA256, actualPath); newInfo0.narHash = narHashAndSize.first; newInfo0.narSize = narHashAndSize.second; @@ -2531,7 +2531,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() std::string { scratchPath->hashPart() }, std::string { requiredFinalPath.hashPart() }); rewriteOutput(outputRewrites); - auto narHashAndSize = hashPath(htSHA256, actualPath); + auto narHashAndSize = hashPath(HashAlgorithm::SHA256, actualPath); ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; newInfo0.narSize = narHashAndSize.second; auto refs = rewriteRefs(); @@ -2546,7 +2546,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating { .method = dof.ca.method, - .hashType = wanted.type, + .hashAlgo = wanted.algo, }); /* Check wanted hash */ @@ -2583,7 +2583,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() [&](const DerivationOutput::Impure & doi) { return newInfoFromCA(DerivationOutput::CAFloating { .method = doi.method, - .hashType = doi.hashType, + .hashAlgo = doi.hashAlgo, }); }, @@ -2945,7 +2945,7 @@ StorePath LocalDerivationGoal::makeFallbackPath(OutputNameView outputName) { return worker.store.makeStorePath( "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName), - Hash(htSHA256), outputPathName(drv->name, outputName)); + Hash(HashAlgorithm::SHA256), outputPathName(drv->name, outputName)); } @@ -2953,7 +2953,7 @@ StorePath LocalDerivationGoal::makeFallbackPath(const StorePath & path) { return worker.store.makeStorePath( "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()), - Hash(htSHA256), path.name()); + Hash(HashAlgorithm::SHA256), path.name()); } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 01f52e7ab..9b8c36286 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -519,8 +519,8 @@ bool Worker::pathContentsGood(const StorePath & path) if (!pathExists(store.printStorePath(path))) res = false; else { - HashResult current = hashPath(info->narHash.type, store.printStorePath(path)); - Hash nullHash(htSHA256); + HashResult current = hashPath(info->narHash.algo, store.printStorePath(path)); + Hash nullHash(HashAlgorithm::SHA256); res = info->narHash == nullHash || info->narHash == current.first; } pathContentsGoodCache.insert_or_assign(path, res); diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 357800333..2086bd0b9 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -63,9 +63,9 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) for (auto hashedMirror : settings.hashedMirrors.get()) try { if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; - std::optional ht = parseHashTypeOpt(getAttr("outputHashAlgo")); + std::optional ht = parseHashAlgoOpt(getAttr("outputHashAlgo")); Hash h = newHashAllowEmpty(getAttr("outputHash"), ht); - fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(HashFormat::Base16, false)); + fetch(hashedMirror + printHashAlgo(h.algo) + "/" + h.to_string(HashFormat::Base16, false)); return; } catch (Error & e) { debug(e.what()); diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index a5f7cdf81..de8194f73 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -38,14 +38,14 @@ ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m) return FileIngestionMethod::Flat; } -std::string ContentAddressMethod::render(HashType ht) const +std::string ContentAddressMethod::render(HashAlgorithm ha) const { return std::visit(overloaded { [&](const TextIngestionMethod & th) { - return std::string{"text:"} + printHashType(ht); + return std::string{"text:"} + printHashAlgo(ha); }, [&](const FileIngestionMethod & fim) { - return "fixed:" + makeFileIngestionPrefix(fim) + printHashType(ht); + return "fixed:" + makeFileIngestionPrefix(fim) + printHashAlgo(ha); } }, raw); } @@ -67,7 +67,7 @@ std::string ContentAddress::render() const /** * Parses content address strings up to the hash. */ -static std::pair parseContentAddressMethodPrefix(std::string_view & rest) +static std::pair parseContentAddressMethodPrefix(std::string_view & rest) { std::string_view wholeInput { rest }; @@ -83,27 +83,27 @@ static std::pair parseContentAddressMethodPrefix auto hashTypeRaw = splitPrefixTo(rest, ':'); if (!hashTypeRaw) throw UsageError("content address hash must be in form ':', but found: %s", wholeInput); - HashType hashType = parseHashType(*hashTypeRaw); - return hashType; + HashAlgorithm hashAlgo = parseHashAlgo(*hashTypeRaw); + return hashAlgo; }; // Switch on prefix if (prefix == "text") { // No parsing of the ingestion method, "text" only support flat. - HashType hashType = parseHashType_(); + HashAlgorithm hashAlgo = parseHashType_(); return { TextIngestionMethod {}, - std::move(hashType), + std::move(hashAlgo), }; } else if (prefix == "fixed") { // Parse method auto method = FileIngestionMethod::Flat; if (splitPrefix(rest, "r:")) method = FileIngestionMethod::Recursive; - HashType hashType = parseHashType_(); + HashAlgorithm hashAlgo = parseHashType_(); return { std::move(method), - std::move(hashType), + std::move(hashAlgo), }; } else throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix); @@ -113,15 +113,15 @@ ContentAddress ContentAddress::parse(std::string_view rawCa) { auto rest = rawCa; - auto [caMethod, hashType] = parseContentAddressMethodPrefix(rest); + auto [caMethod, hashAlgo] = parseContentAddressMethodPrefix(rest); return ContentAddress { .method = std::move(caMethod), - .hash = Hash::parseNonSRIUnprefixed(rest, hashType), + .hash = Hash::parseNonSRIUnprefixed(rest, hashAlgo), }; } -std::pair ContentAddressMethod::parse(std::string_view caMethod) +std::pair ContentAddressMethod::parse(std::string_view caMethod) { std::string asPrefix = std::string{caMethod} + ":"; // parseContentAddressMethodPrefix takes its argument by reference @@ -144,7 +144,7 @@ std::string renderContentAddress(std::optional ca) std::string ContentAddress::printMethodAlgo() const { return method.renderPrefix() - + printHashType(hash.type); + + printHashAlgo(hash.algo); } bool StoreReferences::empty() const diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index bdb558907..05234da38 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -94,7 +94,7 @@ struct ContentAddressMethod /** * Parse a content addressing method and hash type. */ - static std::pair parse(std::string_view rawCaMethod); + static std::pair parse(std::string_view rawCaMethod); /** * Render a content addressing method and hash type in a @@ -102,7 +102,7 @@ struct ContentAddressMethod * * The rough inverse of `parse()`. */ - std::string render(HashType ht) const; + std::string render(HashAlgorithm ha) const; }; diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index be9b0b0d3..530b1a178 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -400,22 +400,22 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto pathInfo = [&]() { // NB: FramedSource must be out of scope before logger->stopWork(); - auto [contentAddressMethod, hashType_] = ContentAddressMethod::parse(camStr); - auto hashType = hashType_; // work around clang bug + auto [contentAddressMethod, hashAlgo_] = ContentAddressMethod::parse(camStr); + auto hashAlgo = hashAlgo_; // work around clang bug FramedSource source(from); // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store. return std::visit(overloaded { [&](const TextIngestionMethod &) { - if (hashType != htSHA256) + if (hashAlgo != HashAlgorithm::SHA256) throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given", - name, printHashType(hashType)); + name, printHashAlgo(hashAlgo)); // We could stream this by changing Store std::string contents = source.drain(); auto path = store->addTextToStore(name, contents, refs, repair); return store->queryPathInfo(path); }, [&](const FileIngestionMethod & fim) { - auto path = store->addToStoreFromDump(source, name, fim, hashType, repair, refs); + auto path = store->addToStoreFromDump(source, name, fim, hashAlgo, repair, refs); return store->queryPathInfo(path); }, }, contentAddressMethod.raw); @@ -424,7 +424,7 @@ static void performOp(TunnelLogger * logger, ref store, WorkerProto::Serialise::write(*store, wconn, *pathInfo); } else { - HashType hashAlgo; + HashAlgorithm hashAlgo; std::string baseName; FileIngestionMethod method; { @@ -440,7 +440,7 @@ static void performOp(TunnelLogger * logger, ref store, hashAlgoRaw = "sha256"; method = FileIngestionMethod::Recursive; } - hashAlgo = parseHashType(hashAlgoRaw); + hashAlgo = parseHashAlgo(hashAlgoRaw); } auto dumpSource = sinkToSource([&](Sink & saved) { @@ -883,7 +883,7 @@ static void performOp(TunnelLogger * logger, ref store, bool repair, dontCheckSigs; auto path = store->parseStorePath(readString(from)); auto deriver = readString(from); - auto narHash = Hash::parseAny(readString(from), htSHA256); + auto narHash = Hash::parseAny(readString(from), HashAlgorithm::SHA256); ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = store->parseStorePath(deriver); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index dd87203b8..c68631c1a 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -215,25 +215,25 @@ static StringSet parseStrings(std::istream & str, bool arePaths) static DerivationOutput parseDerivationOutput( const StoreDirConfig & store, - std::string_view pathS, std::string_view hashAlgo, std::string_view hashS, + std::string_view pathS, std::string_view hashAlgoStr, std::string_view hashS, const ExperimentalFeatureSettings & xpSettings) { - if (hashAlgo != "") { - ContentAddressMethod method = ContentAddressMethod::parsePrefix(hashAlgo); + if (hashAlgoStr != "") { + ContentAddressMethod method = ContentAddressMethod::parsePrefix(hashAlgoStr); if (method == TextIngestionMethod {}) xpSettings.require(Xp::DynamicDerivations); - const auto hashType = parseHashType(hashAlgo); + const auto hashAlgo = parseHashAlgo(hashAlgoStr); if (hashS == "impure") { xpSettings.require(Xp::ImpureDerivations); if (pathS != "") throw FormatError("impure derivation output should not specify output path"); return DerivationOutput::Impure { .method = std::move(method), - .hashType = std::move(hashType), + .hashAlgo = std::move(hashAlgo), }; } else if (hashS != "") { validatePath(pathS); - auto hash = Hash::parseNonSRIUnprefixed(hashS, hashType); + auto hash = Hash::parseNonSRIUnprefixed(hashS, hashAlgo); return DerivationOutput::CAFixed { .ca = ContentAddress { .method = std::move(method), @@ -246,7 +246,7 @@ static DerivationOutput parseDerivationOutput( throw FormatError("content-addressed derivation output should not specify output path"); return DerivationOutput::CAFloating { .method = std::move(method), - .hashType = std::move(hashType), + .hashAlgo = std::move(hashAlgo), }; } } else { @@ -547,7 +547,7 @@ std::string Derivation::unparse(const StoreDirConfig & store, bool maskOutputs, }, [&](const DerivationOutput::CAFloating & dof) { s += ','; printUnquotedString(s, ""); - s += ','; printUnquotedString(s, dof.method.renderPrefix() + printHashType(dof.hashType)); + s += ','; printUnquotedString(s, dof.method.renderPrefix() + printHashAlgo(dof.hashAlgo)); s += ','; printUnquotedString(s, ""); }, [&](const DerivationOutput::Deferred &) { @@ -558,7 +558,7 @@ std::string Derivation::unparse(const StoreDirConfig & store, bool maskOutputs, [&](const DerivationOutput::Impure & doi) { // FIXME s += ','; printUnquotedString(s, ""); - s += ','; printUnquotedString(s, doi.method.renderPrefix() + printHashType(doi.hashType)); + s += ','; printUnquotedString(s, doi.method.renderPrefix() + printHashAlgo(doi.hashAlgo)); s += ','; printUnquotedString(s, "impure"); } }, i.second.raw); @@ -631,7 +631,7 @@ DerivationType BasicDerivation::type() const floatingCAOutputs, deferredIAOutputs, impureOutputs; - std::optional floatingHashType; + std::optional floatingHashAlgo; for (auto & i : outputs) { std::visit(overloaded { @@ -643,10 +643,10 @@ DerivationType BasicDerivation::type() const }, [&](const DerivationOutput::CAFloating & dof) { floatingCAOutputs.insert(i.first); - if (!floatingHashType) { - floatingHashType = dof.hashType; + if (!floatingHashAlgo) { + floatingHashAlgo = dof.hashAlgo; } else { - if (*floatingHashType != dof.hashType) + if (*floatingHashAlgo != dof.hashAlgo) throw Error("all floating outputs must use the same hash type"); } }, @@ -774,7 +774,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut std::map outputHashes; for (const auto & i : drv.outputs) { auto & dof = std::get(i.second.raw); - auto hash = hashString(htSHA256, "fixed:out:" + auto hash = hashString(HashAlgorithm::SHA256, "fixed:out:" + dof.ca.printMethodAlgo() + ":" + dof.ca.hash.to_string(HashFormat::Base16, false) + ":" + store.printStorePath(dof.path(store, drv.name, i.first))); @@ -825,7 +825,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut } } - auto hash = hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2)); + auto hash = hashString(HashAlgorithm::SHA256, drv.unparse(store, maskOutputs, &inputs2)); std::map outputHashes; for (const auto & [outputName, _] : drv.outputs) { @@ -930,7 +930,7 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva }, [&](const DerivationOutput::CAFloating & dof) { out << "" - << (dof.method.renderPrefix() + printHashType(dof.hashType)) + << (dof.method.renderPrefix() + printHashAlgo(dof.hashAlgo)) << ""; }, [&](const DerivationOutput::Deferred &) { @@ -940,7 +940,7 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva }, [&](const DerivationOutput::Impure & doi) { out << "" - << (doi.method.renderPrefix() + printHashType(doi.hashType)) + << (doi.method.renderPrefix() + printHashAlgo(doi.hashAlgo)) << "impure"; }, }, i.second.raw); @@ -958,7 +958,7 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva std::string hashPlaceholder(const OutputNameView outputName) { // FIXME: memoize? - return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(HashFormat::Base32, false); + return "/" + hashString(HashAlgorithm::SHA256, concatStrings("nix-output:", outputName)).to_string(HashFormat::Base32, false); } @@ -1150,7 +1150,7 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const } -const Hash impureOutputHash = hashString(htSHA256, "impure"); +const Hash impureOutputHash = hashString(HashAlgorithm::SHA256, "impure"); nlohmann::json DerivationOutput::toJSON( const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const @@ -1167,11 +1167,11 @@ nlohmann::json DerivationOutput::toJSON( // FIXME print refs? }, [&](const DerivationOutput::CAFloating & dof) { - res["hashAlgo"] = dof.method.renderPrefix() + printHashType(dof.hashType); + res["hashAlgo"] = dof.method.renderPrefix() + printHashAlgo(dof.hashAlgo); }, [&](const DerivationOutput::Deferred &) {}, [&](const DerivationOutput::Impure & doi) { - res["hashAlgo"] = doi.method.renderPrefix() + printHashType(doi.hashType); + res["hashAlgo"] = doi.method.renderPrefix() + printHashAlgo(doi.hashAlgo); res["impure"] = true; }, }, raw); @@ -1191,15 +1191,15 @@ DerivationOutput DerivationOutput::fromJSON( for (const auto & [key, _] : json) keys.insert(key); - auto methodAlgo = [&]() -> std::pair { - std::string hashAlgo = json["hashAlgo"]; + auto methodAlgo = [&]() -> std::pair { + std::string hashAlgoStr = json["hashAlgo"]; // remaining to parse, will be mutated by parsers - std::string_view s = hashAlgo; + std::string_view s = hashAlgoStr; ContentAddressMethod method = ContentAddressMethod::parsePrefix(s); if (method == TextIngestionMethod {}) xpSettings.require(Xp::DynamicDerivations); - auto hashType = parseHashType(s); - return { std::move(method), std::move(hashType) }; + auto hashAlgo = parseHashAlgo(s); + return { std::move(method), std::move(hashAlgo) }; }; if (keys == (std::set { "path" })) { @@ -1209,11 +1209,11 @@ DerivationOutput DerivationOutput::fromJSON( } else if (keys == (std::set { "path", "hashAlgo", "hash" })) { - auto [method, hashType] = methodAlgo(); + auto [method, hashAlgo] = methodAlgo(); auto dof = DerivationOutput::CAFixed { .ca = ContentAddress { .method = std::move(method), - .hash = Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType), + .hash = Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashAlgo), }, }; if (dof.path(store, drvName, outputName) != store.parseStorePath((std::string) json["path"])) @@ -1223,10 +1223,10 @@ DerivationOutput DerivationOutput::fromJSON( else if (keys == (std::set { "hashAlgo" })) { xpSettings.require(Xp::CaDerivations); - auto [method, hashType] = methodAlgo(); + auto [method, hashAlgo] = methodAlgo(); return DerivationOutput::CAFloating { .method = std::move(method), - .hashType = std::move(hashType), + .hashAlgo = std::move(hashAlgo), }; } @@ -1236,10 +1236,10 @@ DerivationOutput DerivationOutput::fromJSON( else if (keys == (std::set { "hashAlgo", "impure" })) { xpSettings.require(Xp::ImpureDerivations); - auto [method, hashType] = methodAlgo(); + auto [method, hashAlgo] = methodAlgo(); return DerivationOutput::Impure { .method = std::move(method), - .hashType = hashType, + .hashAlgo = hashAlgo, }; } diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 219e8e7d7..290abedcf 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -75,9 +75,9 @@ struct DerivationOutput /** * How the serialization will be hashed */ - HashType hashType; + HashAlgorithm hashAlgo; - GENERATE_CMP(CAFloating, me->method, me->hashType); + GENERATE_CMP(CAFloating, me->method, me->hashAlgo); }; /** @@ -102,9 +102,9 @@ struct DerivationOutput /** * How the serialization will be hashed */ - HashType hashType; + HashAlgorithm hashAlgo; - GENERATE_CMP(Impure, me->method, me->hashType); + GENERATE_CMP(Impure, me->method, me->hashAlgo); }; typedef std::variant< diff --git a/src/libstore/downstream-placeholder.cc b/src/libstore/downstream-placeholder.cc index ca9f7476e..10df37fa4 100644 --- a/src/libstore/downstream-placeholder.cc +++ b/src/libstore/downstream-placeholder.cc @@ -19,7 +19,7 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownCaOutput( auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4); auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName); return DownstreamPlaceholder { - hashString(htSHA256, clearText) + hashString(HashAlgorithm::SHA256, clearText) }; } @@ -34,7 +34,7 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation( + compressed.to_string(HashFormat::Base32, false) + ":" + std::string { outputName }; return DownstreamPlaceholder { - hashString(htSHA256, clearText) + hashString(HashAlgorithm::SHA256, clearText) }; } diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 52130f8f6..48718ef84 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -30,7 +30,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) { auto info = queryPathInfo(path); - HashSink hashSink(htSHA256); + HashSink hashSink(HashAlgorithm::SHA256); TeeSink teeSink(sink, hashSink); narFromPath(path, teeSink); @@ -39,7 +39,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) filesystem corruption from spreading to other machines. Don't complain if the stored hash is zero (unknown). */ Hash hash = hashSink.currentHash().first; - if (hash != info->narHash && info->narHash != Hash(info->narHash.type)) + if (hash != info->narHash && info->narHash != Hash(info->narHash.algo)) throw Error("hash of path '%s' has changed from '%s' to '%s'!", printStorePath(path), info->narHash.to_string(HashFormat::Base32, true), hash.to_string(HashFormat::Base32, true)); @@ -79,7 +79,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) auto references = CommonProto::Serialise::read(*this, CommonProto::ReadConn { .from = source }); auto deriver = readString(source); - auto narHash = hashString(htSHA256, saved.s); + auto narHash = hashString(HashAlgorithm::SHA256, saved.s); ValidPathInfo info { path, narHash }; if (deriver != "") diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 93fa60682..5c413aa77 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -50,7 +50,7 @@ static void makeSymlink(const Path & link, const Path & target) void LocalStore::addIndirectRoot(const Path & path) { - std::string hash = hashString(htSHA1, path).to_string(HashFormat::Base32, false); + std::string hash = hashString(HashAlgorithm::SHA1, path).to_string(HashFormat::Base32, false); Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", stateDir, gcRootsDir, hash)); makeSymlink(realRoot, path); } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 731457354..fb1580dd6 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -267,13 +267,13 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor { unsupported("queryPathFromHashPart"); } StorePath addToStore( - std::string_view name, - const Path & srcPath, - FileIngestionMethod method, - HashType hashAlgo, - PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) override + std::string_view name, + const Path & srcPath, + FileIngestionMethod method, + HashAlgorithm hashAlgo, + PathFilter & filter, + RepairFlag repair, + const StorePathSet & references) override { unsupported("addToStore"); } StorePath addTextToStore( diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c8962f574..ef7dd7985 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -955,7 +955,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) StorePathSet paths; for (auto & [_, i] : infos) { - assert(i.narHash.type == htSHA256); + assert(i.narHash.algo == HashAlgorithm::SHA256); if (isValidPath_(*state, i.path)) updatePathInfo(*state, i); else @@ -1069,7 +1069,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, /* While restoring the path from the NAR, compute the hash of the NAR. */ - HashSink hashSink(htSHA256); + HashSink hashSink(HashAlgorithm::SHA256); TeeSource wrapperSource { source, hashSink }; @@ -1090,7 +1090,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, auto & specified = *info.ca; auto actualHash = hashCAPath( specified.method, - specified.hash.type, + specified.hash.algo, info.path ); if (specified.hash != actualHash.hash) { @@ -1116,7 +1116,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) + FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) { /* For computing the store path. */ auto hashSink = std::make_unique(hashAlgo); @@ -1220,8 +1220,8 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name /* For computing the nar hash. In recursive SHA-256 mode, this is the same as the store hash, so no need to do it again. */ auto narHash = std::pair { hash, size }; - if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) { - HashSink narSink { htSHA256 }; + if (method != FileIngestionMethod::Recursive || hashAlgo != HashAlgorithm::SHA256) { + HashSink narSink { HashAlgorithm::SHA256 }; dumpPath(realPath, narSink); narHash = narSink.finish(); } @@ -1252,7 +1252,7 @@ StorePath LocalStore::addTextToStore( std::string_view s, const StorePathSet & references, RepairFlag repair) { - auto hash = hashString(htSHA256, s); + auto hash = hashString(HashAlgorithm::SHA256, s); auto dstPath = makeTextPath(name, TextInfo { .hash = hash, .references = references, @@ -1278,7 +1278,7 @@ StorePath LocalStore::addTextToStore( StringSink sink; dumpString(s, sink); - auto narHash = hashString(htSHA256, sink.s); + auto narHash = hashString(HashAlgorithm::SHA256, sink.s); optimisePath(realPath, repair); @@ -1389,7 +1389,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) for (auto & link : readDirectory(linksDir)) { printMsg(lvlTalkative, "checking contents of '%s'", link.name); Path linkPath = linksDir + "/" + link.name; - std::string hash = hashPath(htSHA256, linkPath).first.to_string(HashFormat::Base32, false); + std::string hash = hashPath(HashAlgorithm::SHA256, linkPath).first.to_string(HashFormat::Base32, false); if (hash != link.name) { printError("link '%s' was modified! expected hash '%s', got '%s'", linkPath, link.name, hash); @@ -1406,7 +1406,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) printInfo("checking store hashes..."); - Hash nullHash(htSHA256); + Hash nullHash(HashAlgorithm::SHA256); for (auto & i : validPaths) { try { @@ -1415,7 +1415,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) /* Check the content hash (optionally - slow). */ printMsg(lvlTalkative, "checking contents of '%s'", printStorePath(i)); - auto hashSink = HashSink(info->narHash.type); + auto hashSink = HashSink(info->narHash.algo); dumpPath(Store::toRealPath(i), hashSink); auto current = hashSink.finish(); @@ -1697,20 +1697,20 @@ void LocalStore::queryRealisationUncached(const DrvOutput & id, } ContentAddress LocalStore::hashCAPath( - const ContentAddressMethod & method, const HashType & hashType, + const ContentAddressMethod & method, const HashAlgorithm & hashAlgo, const StorePath & path) { - return hashCAPath(method, hashType, Store::toRealPath(path), path.hashPart()); + return hashCAPath(method, hashAlgo, Store::toRealPath(path), path.hashPart()); } ContentAddress LocalStore::hashCAPath( const ContentAddressMethod & method, - const HashType & hashType, + const HashAlgorithm & hashAlgo, const Path & path, const std::string_view pathHash ) { - HashModuloSink caSink ( hashType, std::string(pathHash) ); + HashModuloSink caSink ( hashAlgo, std::string(pathHash) ); std::visit(overloaded { [&](const TextIngestionMethod &) { readFile(path, caSink); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 8f0ffd2a2..ee605b5a2 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -178,7 +178,7 @@ public: RepairFlag repair, CheckSigsFlag checkSigs) override; StorePath addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override; + FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) override; StorePath addTextToStore( std::string_view name, @@ -353,12 +353,12 @@ private: // XXX: Make a generic `Store` method ContentAddress hashCAPath( const ContentAddressMethod & method, - const HashType & hashType, + const HashAlgorithm & hashAlgo, const StorePath & path); ContentAddress hashCAPath( const ContentAddressMethod & method, - const HashType & hashType, + const HashAlgorithm & hashAlgo, const Path & path, const std::string_view pathHash ); diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 253609ed2..170fe67b9 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -43,7 +43,7 @@ std::map makeContentAddressed( sink.s = rewriteStrings(sink.s, rewrites); - HashModuloSink hashModuloSink(htSHA256, oldHashPart); + HashModuloSink hashModuloSink(HashAlgorithm::SHA256, oldHashPart); hashModuloSink(sink.s); auto narModuloHash = hashModuloSink.finish().first; @@ -66,7 +66,7 @@ std::map makeContentAddressed( rsink2(sink.s); rsink2.flush(); - info.narHash = hashString(htSHA256, sink2.s); + info.narHash = hashString(HashAlgorithm::SHA256, sink2.s); info.narSize = sink.s.size(); StringSource source(sink2.s); diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 1060a6c8b..25e2a7d7b 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -113,10 +113,10 @@ std::string NarInfo::to_string(const Store & store) const res += "URL: " + url + "\n"; assert(compression != ""); res += "Compression: " + compression + "\n"; - assert(fileHash && fileHash->type == htSHA256); + assert(fileHash && fileHash->algo == HashAlgorithm::SHA256); res += "FileHash: " + fileHash->to_string(HashFormat::Base32, true) + "\n"; res += "FileSize: " + std::to_string(fileSize) + "\n"; - assert(narHash.type == htSHA256); + assert(narHash.algo == HashAlgorithm::SHA256); res += "NarHash: " + narHash.to_string(HashFormat::Base32, true) + "\n"; res += "NarSize: " + std::to_string(narSize) + "\n"; diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 0fa977545..cadf88347 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -146,7 +146,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, Also note that if `path' is a symlink, then we're hashing the contents of the symlink (i.e. the result of readlink()), not the contents of the target (which may not even exist). */ - Hash hash = hashPath(htSHA256, path).first; + Hash hash = hashPath(HashAlgorithm::SHA256, path).first; debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Base32, true)); /* Check if this is a known hash. */ @@ -156,7 +156,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, if (pathExists(linkPath)) { auto stLink = lstat(linkPath); if (st.st_size != stLink.st_size - || (repair && hash != hashPath(htSHA256, linkPath).first)) + || (repair && hash != hashPath(HashAlgorithm::SHA256, linkPath).first)) { // XXX: Consider overwriting linkPath with our valid version. warn("removing corrupted link '%s'", linkPath); diff --git a/src/libstore/path-references.cc b/src/libstore/path-references.cc index 274b596c0..15f52ec9d 100644 --- a/src/libstore/path-references.cc +++ b/src/libstore/path-references.cc @@ -49,7 +49,7 @@ std::pair scanForReferences( const std::string & path, const StorePathSet & refs) { - HashSink hashSink { htSHA256 }; + HashSink hashSink { HashAlgorithm::SHA256 }; auto found = scanForReferences(hashSink, path, refs); auto hash = hashSink.finish(); return std::pair(found, hash); diff --git a/src/libstore/path.cc b/src/libstore/path.cc index 69f6d7356..d5257c939 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -49,7 +49,7 @@ StorePath StorePath::dummy("ffffffffffffffffffffffffffffffff-x"); StorePath StorePath::random(std::string_view name) { - Hash hash(htSHA1); + Hash hash(HashAlgorithm::SHA1); randombytes_buf(hash.hash, hash.hashSize); return StorePath(hash, name); } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 3d3919882..cc26c2a94 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -417,12 +417,12 @@ std::optional RemoteStore::queryPathFromHashPart(const std::string & ref RemoteStore::addCAToStore( - Source & dump, - std::string_view name, - ContentAddressMethod caMethod, - HashType hashType, - const StorePathSet & references, - RepairFlag repair) + Source & dump, + std::string_view name, + ContentAddressMethod caMethod, + HashAlgorithm hashAlgo, + const StorePathSet & references, + RepairFlag repair) { std::optional conn_(getConnection()); auto & conn = *conn_; @@ -432,7 +432,7 @@ ref RemoteStore::addCAToStore( conn->to << WorkerProto::Op::AddToStore << name - << caMethod.render(hashType); + << caMethod.render(hashAlgo); WorkerProto::write(*this, *conn, references); conn->to << repair; @@ -453,9 +453,9 @@ ref RemoteStore::addCAToStore( std::visit(overloaded { [&](const TextIngestionMethod & thm) -> void { - if (hashType != htSHA256) + if (hashAlgo != HashAlgorithm::SHA256) throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given", - name, printHashType(hashType)); + name, printHashAlgo(hashAlgo)); std::string s = dump.drain(); conn->to << WorkerProto::Op::AddTextToStore << name << s; WorkerProto::write(*this, *conn, references); @@ -465,9 +465,9 @@ ref RemoteStore::addCAToStore( conn->to << WorkerProto::Op::AddToStore << name - << ((hashType == htSHA256 && fim == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ + << ((hashAlgo == HashAlgorithm::SHA256 && fim == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ << (fim == FileIngestionMethod::Recursive ? 1 : 0) - << printHashType(hashType); + << printHashAlgo(hashAlgo); try { conn->to.written = 0; @@ -503,9 +503,9 @@ ref RemoteStore::addCAToStore( StorePath RemoteStore::addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method, HashType hashType, RepairFlag repair, const StorePathSet & references) + FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) { - return addCAToStore(dump, name, method, hashType, references, repair)->path; + return addCAToStore(dump, name, method, hashAlgo, references, repair)->path; } @@ -610,7 +610,7 @@ StorePath RemoteStore::addTextToStore( RepairFlag repair) { StringSource source(s); - return addCAToStore(source, name, TextIngestionMethod {}, htSHA256, references, repair)->path; + return addCAToStore(source, name, TextIngestionMethod {}, HashAlgorithm::SHA256, references, repair)->path; } void RemoteStore::registerDrvOutput(const Realisation & info) diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 1cc11af86..f2e34c1a3 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -74,18 +74,18 @@ public: * Add a content-addressable store path. `dump` will be drained. */ ref addCAToStore( - Source & dump, - std::string_view name, - ContentAddressMethod caMethod, - HashType hashType, - const StorePathSet & references, - RepairFlag repair); + Source & dump, + std::string_view name, + ContentAddressMethod caMethod, + HashAlgorithm hashAlgo, + const StorePathSet & references, + RepairFlag repair); /** * Add a content-addressable store path. Does not support references. `dump` will be drained. */ StorePath addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) override; + FileIngestionMethod method = FileIngestionMethod::Recursive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) override; void addToStore(const ValidPathInfo & info, Source & nar, RepairFlag repair, CheckSigsFlag checkSigs) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 8601e0857..800df7fa0 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -153,7 +153,7 @@ StorePath StoreDirConfig::makeStorePath(std::string_view type, /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ auto s = std::string(type) + ":" + std::string(hash) + ":" + storeDir + ":" + std::string(name); - auto h = compressHash(hashString(htSHA256, s), 20); + auto h = compressHash(hashString(HashAlgorithm::SHA256, s), 20); return StorePath(h, name); } @@ -191,12 +191,12 @@ static std::string makeType( StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { - if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) { + if (info.hash.algo == HashAlgorithm::SHA256 && info.method == FileIngestionMethod::Recursive) { return makeStorePath(makeType(*this, "source", info.references), info.hash, name); } else { assert(info.references.size() == 0); return makeStorePath("output:out", - hashString(htSHA256, + hashString(HashAlgorithm::SHA256, "fixed:out:" + makeFileIngestionPrefix(info.method) + info.hash.to_string(HashFormat::Base16, true) + ":"), @@ -207,7 +207,7 @@ StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const Fixed StorePath StoreDirConfig::makeTextPath(std::string_view name, const TextInfo & info) const { - assert(info.hash.type == htSHA256); + assert(info.hash.algo == HashAlgorithm::SHA256); return makeStorePath( makeType(*this, "text", StoreReferences { .others = info.references, @@ -233,11 +233,11 @@ StorePath StoreDirConfig::makeFixedOutputPathFromCA(std::string_view name, const std::pair StoreDirConfig::computeStorePathFromDump( - Source & dump, - std::string_view name, - FileIngestionMethod method, - HashType hashAlgo, - const StorePathSet & references) const + Source & dump, + std::string_view name, + FileIngestionMethod method, + HashAlgorithm hashAlgo, + const StorePathSet & references) const { HashSink sink(hashAlgo); dump.drainInto(sink); @@ -257,20 +257,20 @@ StorePath StoreDirConfig::computeStorePathForText( const StorePathSet & references) const { return makeTextPath(name, TextInfo { - .hash = hashString(htSHA256, s), + .hash = hashString(HashAlgorithm::SHA256, s), .references = references, }); } StorePath Store::addToStore( - std::string_view name, - const Path & _srcPath, - FileIngestionMethod method, - HashType hashAlgo, - PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) + std::string_view name, + const Path & _srcPath, + FileIngestionMethod method, + HashAlgorithm hashAlgo, + PathFilter & filter, + RepairFlag repair, + const StorePathSet & references) { Path srcPath(absPath(_srcPath)); auto source = sinkToSource([&](Sink & sink) { @@ -405,10 +405,10 @@ digraph graphname { } */ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, - FileIngestionMethod method, HashType hashAlgo, - std::optional expectedCAHash) + FileIngestionMethod method, HashAlgorithm hashAlgo, + std::optional expectedCAHash) { - HashSink narHashSink { htSHA256 }; + HashSink narHashSink { HashAlgorithm::SHA256 }; HashSink caHashSink { hashAlgo }; /* Note that fileSink and unusualHashTee must be mutually exclusive, since @@ -417,7 +417,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, RegularFileSink fileSink { caHashSink }; TeeSink unusualHashTee { narHashSink, caHashSink }; - auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != htSHA256 + auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != HashAlgorithm::SHA256 ? static_cast(unusualHashTee) : narHashSink; @@ -445,7 +445,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, finish. */ auto [narHash, narSize] = narHashSink.finish(); - auto hash = method == FileIngestionMethod::Recursive && hashAlgo == htSHA256 + auto hash = method == FileIngestionMethod::Recursive && hashAlgo == HashAlgorithm::SHA256 ? narHash : caHashSink.finish().first; @@ -1205,7 +1205,7 @@ std::optional decodeValidPathInfo(const Store & store, std::istre if (!hashGiven) { std::string s; getline(str, s); - auto narHash = Hash::parseAny(s, htSHA256); + auto narHash = Hash::parseAny(s, HashAlgorithm::SHA256); getline(str, s); auto narSize = string2Int(s); if (!narSize) throw Error("number expected"); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 5860d0ea6..ada6699d5 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -427,13 +427,13 @@ public: * libutil/archive.hh). */ virtual StorePath addToStore( - std::string_view name, - const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, - HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, - RepairFlag repair = NoRepair, - const StorePathSet & references = StorePathSet()); + std::string_view name, + const Path & srcPath, + FileIngestionMethod method = FileIngestionMethod::Recursive, + HashAlgorithm hashAlgo = HashAlgorithm::SHA256, + PathFilter & filter = defaultPathFilter, + RepairFlag repair = NoRepair, + const StorePathSet & references = StorePathSet()); /** * Copy the contents of a path to the store and register the @@ -441,8 +441,8 @@ public: * memory. */ ValidPathInfo addToStoreSlow(std::string_view name, const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, - std::optional expectedCAHash = {}); + FileIngestionMethod method = FileIngestionMethod::Recursive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, + std::optional expectedCAHash = {}); /** * Like addToStore(), but the contents of the path are contained @@ -454,8 +454,8 @@ public: * \todo remove? */ virtual StorePath addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, - const StorePathSet & references = StorePathSet()) + FileIngestionMethod method = FileIngestionMethod::Recursive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, RepairFlag repair = NoRepair, + const StorePathSet & references = StorePathSet()) { unsupported("addToStoreFromDump"); } /** diff --git a/src/libstore/store-dir-config.hh b/src/libstore/store-dir-config.hh index 53843d663..8dafca096 100644 --- a/src/libstore/store-dir-config.hh +++ b/src/libstore/store-dir-config.hh @@ -98,7 +98,7 @@ struct StoreDirConfig : public Config Source & dump, std::string_view name, FileIngestionMethod method = FileIngestionMethod::Recursive, - HashType hashAlgo = htSHA256, + HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = {}) const; /** diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index 43654d7e8..2a379e75e 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -160,7 +160,7 @@ void WorkerProto::Serialise::write(const StoreDirConfig & store, UnkeyedValidPathInfo WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) { auto deriver = readString(conn.from); - auto narHash = Hash::parseAny(readString(conn.from), htSHA256); + auto narHash = Hash::parseAny(readString(conn.from), HashAlgorithm::SHA256); UnkeyedValidPathInfo info(narHash); if (deriver != "") info.deriver = store.parseStorePath(deriver); info.references = WorkerProto::Serialise::read(store, conn); diff --git a/src/libutil/args.cc b/src/libutil/args.cc index c4b2975ee..ac3727d11 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -546,32 +546,32 @@ nlohmann::json Args::toJSON() static void hashTypeCompleter(AddCompletions & completions, size_t index, std::string_view prefix) { - for (auto & type : hashTypes) + for (auto & type : hashAlgorithms) if (hasPrefix(type, prefix)) completions.add(type); } -Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht) +Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashAlgorithm * ha) { return Flag { .longName = std::move(longName), .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')", .labels = {"hash-algo"}, - .handler = {[ht](std::string s) { - *ht = parseHashType(s); + .handler = {[ha](std::string s) { + *ha = parseHashAlgo(s); }}, .completer = hashTypeCompleter, }; } -Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional * oht) +Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional * oha) { return Flag { .longName = std::move(longName), .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512'). Optional as can also be gotten from SRI hash itself.", .labels = {"hash-algo"}, - .handler = {[oht](std::string s) { - *oht = std::optional { parseHashType(s) }; + .handler = {[oha](std::string s) { + *oha = std::optional {parseHashAlgo(s) }; }}, .completer = hashTypeCompleter, }; diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 72278dccc..0cff76158 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -14,7 +14,7 @@ namespace nix { -enum HashType : char; +enum struct HashAlgorithm : char; class MultiCommand; @@ -175,8 +175,8 @@ protected: std::optional experimentalFeature; - static Flag mkHashTypeFlag(std::string && longName, HashType * ht); - static Flag mkHashTypeOptFlag(std::string && longName, std::optional * oht); + static Flag mkHashTypeFlag(std::string && longName, HashAlgorithm * ha); + static Flag mkHashTypeOptFlag(std::string && longName, std::optional * oha); }; /** diff --git a/src/libutil/git.cc b/src/libutil/git.cc index a4bd60096..296b75628 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -106,7 +106,7 @@ void parse( std::string hashs = getString(source, 20); left -= 20; - Hash hash(htSHA1); + Hash hash(HashAlgorithm::SHA1); std::copy(hashs.begin(), hashs.end(), hash.hash); hook(name, TreeEntry { @@ -241,12 +241,12 @@ Mode dump( TreeEntry dumpHash( - HashType ht, - SourceAccessor & accessor, const CanonPath & path, PathFilter & filter) + HashAlgorithm ha, + SourceAccessor & accessor, const CanonPath & path, PathFilter & filter) { std::function hook; hook = [&](const CanonPath & path) -> TreeEntry { - auto hashSink = HashSink(ht); + auto hashSink = HashSink(ha); auto mode = dump(accessor, path, hashSink, hook, filter); auto hash = hashSink.finish().first; return { diff --git a/src/libutil/git.hh b/src/libutil/git.hh index 303460072..b24b25dd3 100644 --- a/src/libutil/git.hh +++ b/src/libutil/git.hh @@ -123,9 +123,9 @@ Mode dump( * A smaller wrapper around `dump`. */ TreeEntry dumpHash( - HashType ht, - SourceAccessor & accessor, const CanonPath & path, - PathFilter & filter = defaultPathFilter); + HashAlgorithm ha, + SourceAccessor & accessor, const CanonPath & path, + PathFilter & filter = defaultPathFilter); /** * A line from the output of `git ls-remote --symref`. diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 144f7ae7e..38a29c459 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -16,23 +16,23 @@ namespace nix { -static size_t regularHashSize(HashType type) { +static size_t regularHashSize(HashAlgorithm type) { switch (type) { - case htMD5: return md5HashSize; - case htSHA1: return sha1HashSize; - case htSHA256: return sha256HashSize; - case htSHA512: return sha512HashSize; + case HashAlgorithm::MD5: return md5HashSize; + case HashAlgorithm::SHA1: return sha1HashSize; + case HashAlgorithm::SHA256: return sha256HashSize; + case HashAlgorithm::SHA512: return sha512HashSize; } abort(); } -std::set hashTypes = { "md5", "sha1", "sha256", "sha512" }; +std::set hashAlgorithms = {"md5", "sha1", "sha256", "sha512" }; -Hash::Hash(HashType type) : type(type) +Hash::Hash(HashAlgorithm algo) : algo(algo) { - hashSize = regularHashSize(type); + hashSize = regularHashSize(algo); assert(hashSize <= maxHashSize); memset(hash, 0, maxHashSize); } @@ -109,16 +109,16 @@ static std::string printHash32(const Hash & hash) std::string printHash16or32(const Hash & hash) { - assert(hash.type); - return hash.to_string(hash.type == htMD5 ? HashFormat::Base16 : HashFormat::Base32, false); + assert(static_cast(hash.algo)); + return hash.to_string(hash.algo == HashAlgorithm::MD5 ? HashFormat::Base16 : HashFormat::Base32, false); } -std::string Hash::to_string(HashFormat hashFormat, bool includeType) const +std::string Hash::to_string(HashFormat hashFormat, bool includeAlgo) const { std::string s; - if (hashFormat == HashFormat::SRI || includeType) { - s += printHashType(type); + if (hashFormat == HashFormat::SRI || includeAlgo) { + s += printHashAlgo(algo); s += hashFormat == HashFormat::SRI ? '-' : ':'; } switch (hashFormat) { @@ -136,7 +136,7 @@ std::string Hash::to_string(HashFormat hashFormat, bool includeType) const return s; } -Hash Hash::dummy(htSHA256); +Hash Hash::dummy(HashAlgorithm::SHA256); Hash Hash::parseSRI(std::string_view original) { auto rest = original; @@ -145,18 +145,18 @@ Hash Hash::parseSRI(std::string_view original) { auto hashRaw = splitPrefixTo(rest, '-'); if (!hashRaw) throw BadHash("hash '%s' is not SRI", original); - HashType parsedType = parseHashType(*hashRaw); + HashAlgorithm parsedType = parseHashAlgo(*hashRaw); return Hash(rest, parsedType, true); } // Mutates the string to eliminate the prefixes when found -static std::pair, bool> getParsedTypeAndSRI(std::string_view & rest) +static std::pair, bool> getParsedTypeAndSRI(std::string_view & rest) { bool isSRI = false; // Parse the hash type before the separator, if there was one. - std::optional optParsedType; + std::optional optParsedType; { auto hashRaw = splitPrefixTo(rest, ':'); @@ -166,7 +166,7 @@ static std::pair, bool> getParsedTypeAndSRI(std::string_ isSRI = true; } if (hashRaw) - optParsedType = parseHashType(*hashRaw); + optParsedType = parseHashAlgo(*hashRaw); } return {optParsedType, isSRI}; @@ -185,29 +185,29 @@ Hash Hash::parseAnyPrefixed(std::string_view original) return Hash(rest, *optParsedType, isSRI); } -Hash Hash::parseAny(std::string_view original, std::optional optType) +Hash Hash::parseAny(std::string_view original, std::optional optAlgo) { auto rest = original; auto [optParsedType, isSRI] = getParsedTypeAndSRI(rest); // Either the string or user must provide the type, if they both do they // must agree. - if (!optParsedType && !optType) + if (!optParsedType && !optAlgo) throw BadHash("hash '%s' does not include a type, nor is the type otherwise known from context", rest); - else if (optParsedType && optType && *optParsedType != *optType) - throw BadHash("hash '%s' should have type '%s'", original, printHashType(*optType)); + else if (optParsedType && optAlgo && *optParsedType != *optAlgo) + throw BadHash("hash '%s' should have type '%s'", original, printHashAlgo(*optAlgo)); - HashType hashType = optParsedType ? *optParsedType : *optType; - return Hash(rest, hashType, isSRI); + HashAlgorithm hashAlgo = optParsedType ? *optParsedType : *optAlgo; + return Hash(rest, hashAlgo, isSRI); } -Hash Hash::parseNonSRIUnprefixed(std::string_view s, HashType type) +Hash Hash::parseNonSRIUnprefixed(std::string_view s, HashAlgorithm algo) { - return Hash(s, type, false); + return Hash(s, algo, false); } -Hash::Hash(std::string_view rest, HashType type, bool isSRI) - : Hash(type) +Hash::Hash(std::string_view rest, HashAlgorithm algo, bool isSRI) + : Hash(algo) { if (!isSRI && rest.size() == base16Len()) { @@ -257,19 +257,19 @@ Hash::Hash(std::string_view rest, HashType type, bool isSRI) } else - throw BadHash("hash '%s' has wrong length for hash type '%s'", rest, printHashType(this->type)); + throw BadHash("hash '%s' has wrong length for hash algorithm '%s'", rest, printHashAlgo(this->algo)); } -Hash newHashAllowEmpty(std::string_view hashStr, std::optional ht) +Hash newHashAllowEmpty(std::string_view hashStr, std::optional ha) { if (hashStr.empty()) { - if (!ht) + if (!ha) throw BadHash("empty hash requires explicit hash type"); - Hash h(*ht); + Hash h(*ha); warn("found empty hash, assuming '%s'", h.to_string(HashFormat::SRI, true)); return h; } else - return Hash::parseAny(hashStr, ht); + return Hash::parseAny(hashStr, ha); } @@ -282,58 +282,58 @@ union Ctx }; -static void start(HashType ht, Ctx & ctx) +static void start(HashAlgorithm ha, Ctx & ctx) { - if (ht == htMD5) MD5_Init(&ctx.md5); - else if (ht == htSHA1) SHA1_Init(&ctx.sha1); - else if (ht == htSHA256) SHA256_Init(&ctx.sha256); - else if (ht == htSHA512) SHA512_Init(&ctx.sha512); + if (ha == HashAlgorithm::MD5) MD5_Init(&ctx.md5); + else if (ha == HashAlgorithm::SHA1) SHA1_Init(&ctx.sha1); + else if (ha == HashAlgorithm::SHA256) SHA256_Init(&ctx.sha256); + else if (ha == HashAlgorithm::SHA512) SHA512_Init(&ctx.sha512); } -static void update(HashType ht, Ctx & ctx, - std::string_view data) +static void update(HashAlgorithm ha, Ctx & ctx, + std::string_view data) { - if (ht == htMD5) MD5_Update(&ctx.md5, data.data(), data.size()); - else if (ht == htSHA1) SHA1_Update(&ctx.sha1, data.data(), data.size()); - else if (ht == htSHA256) SHA256_Update(&ctx.sha256, data.data(), data.size()); - else if (ht == htSHA512) SHA512_Update(&ctx.sha512, data.data(), data.size()); + if (ha == HashAlgorithm::MD5) MD5_Update(&ctx.md5, data.data(), data.size()); + else if (ha == HashAlgorithm::SHA1) SHA1_Update(&ctx.sha1, data.data(), data.size()); + else if (ha == HashAlgorithm::SHA256) SHA256_Update(&ctx.sha256, data.data(), data.size()); + else if (ha == HashAlgorithm::SHA512) SHA512_Update(&ctx.sha512, data.data(), data.size()); } -static void finish(HashType ht, Ctx & ctx, unsigned char * hash) +static void finish(HashAlgorithm ha, Ctx & ctx, unsigned char * hash) { - if (ht == htMD5) MD5_Final(hash, &ctx.md5); - else if (ht == htSHA1) SHA1_Final(hash, &ctx.sha1); - else if (ht == htSHA256) SHA256_Final(hash, &ctx.sha256); - else if (ht == htSHA512) SHA512_Final(hash, &ctx.sha512); + if (ha == HashAlgorithm::MD5) MD5_Final(hash, &ctx.md5); + else if (ha == HashAlgorithm::SHA1) SHA1_Final(hash, &ctx.sha1); + else if (ha == HashAlgorithm::SHA256) SHA256_Final(hash, &ctx.sha256); + else if (ha == HashAlgorithm::SHA512) SHA512_Final(hash, &ctx.sha512); } -Hash hashString(HashType ht, std::string_view s) +Hash hashString(HashAlgorithm ha, std::string_view s) { Ctx ctx; - Hash hash(ht); - start(ht, ctx); - update(ht, ctx, s); - finish(ht, ctx, hash.hash); + Hash hash(ha); + start(ha, ctx); + update(ha, ctx, s); + finish(ha, ctx, hash.hash); return hash; } -Hash hashFile(HashType ht, const Path & path) +Hash hashFile(HashAlgorithm ha, const Path & path) { - HashSink sink(ht); + HashSink sink(ha); readFile(path, sink); return sink.finish().first; } -HashSink::HashSink(HashType ht) : ht(ht) +HashSink::HashSink(HashAlgorithm ha) : ha(ha) { ctx = new Ctx; bytes = 0; - start(ht, *ctx); + start(ha, *ctx); } HashSink::~HashSink() @@ -345,14 +345,14 @@ HashSink::~HashSink() void HashSink::writeUnbuffered(std::string_view data) { bytes += data.size(); - update(ht, *ctx, data); + update(ha, *ctx, data); } HashResult HashSink::finish() { flush(); - Hash hash(ht); - nix::finish(ht, *ctx, hash.hash); + Hash hash(ha); + nix::finish(ha, *ctx, hash.hash); return HashResult(hash, bytes); } @@ -360,16 +360,16 @@ HashResult HashSink::currentHash() { flush(); Ctx ctx2 = *ctx; - Hash hash(ht); - nix::finish(ht, ctx2, hash.hash); + Hash hash(ha); + nix::finish(ha, ctx2, hash.hash); return HashResult(hash, bytes); } HashResult hashPath( - HashType ht, const Path & path, PathFilter & filter) + HashAlgorithm ha, const Path & path, PathFilter & filter) { - HashSink sink(ht); + HashSink sink(ha); dumpPath(path, sink, filter); return sink.finish(); } @@ -377,7 +377,7 @@ HashResult hashPath( Hash compressHash(const Hash & hash, unsigned int newSize) { - Hash h(hash.type); + Hash h(hash.algo); h.hashSize = newSize; for (unsigned int i = 0; i < hash.hashSize; ++i) h.hash[i % newSize] ^= hash.hash[i]; @@ -420,31 +420,31 @@ std::string_view printHashFormat(HashFormat HashFormat) } } -std::optional parseHashTypeOpt(std::string_view s) +std::optional parseHashAlgoOpt(std::string_view s) { - if (s == "md5") return htMD5; - if (s == "sha1") return htSHA1; - if (s == "sha256") return htSHA256; - if (s == "sha512") return htSHA512; + if (s == "md5") return HashAlgorithm::MD5; + if (s == "sha1") return HashAlgorithm::SHA1; + if (s == "sha256") return HashAlgorithm::SHA256; + if (s == "sha512") return HashAlgorithm::SHA512; return std::nullopt; } -HashType parseHashType(std::string_view s) +HashAlgorithm parseHashAlgo(std::string_view s) { - auto opt_h = parseHashTypeOpt(s); + auto opt_h = parseHashAlgoOpt(s); if (opt_h) return *opt_h; else throw UsageError("unknown hash algorithm '%1%', expect 'md5', 'sha1', 'sha256', or 'sha512'", s); } -std::string_view printHashType(HashType ht) +std::string_view printHashAlgo(HashAlgorithm ha) { - switch (ht) { - case htMD5: return "md5"; - case htSHA1: return "sha1"; - case htSHA256: return "sha256"; - case htSHA512: return "sha512"; + switch (ha) { + case HashAlgorithm::MD5: return "md5"; + case HashAlgorithm::SHA1: return "sha1"; + case HashAlgorithm::SHA256: return "sha256"; + case HashAlgorithm::SHA512: return "sha512"; default: // illegal hash type enum value internally, as opposed to external input // which should be validated with nice error message. diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 820154e7a..3c97ed4b1 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -12,7 +12,7 @@ namespace nix { MakeError(BadHash, Error); -enum HashType : char { htMD5 = 42, htSHA1, htSHA256, htSHA512 }; +enum struct HashAlgorithm : char { MD5 = 42, SHA1, SHA256, SHA512 }; const int md5HashSize = 16; @@ -20,7 +20,7 @@ const int sha1HashSize = 20; const int sha256HashSize = 32; const int sha512HashSize = 64; -extern std::set hashTypes; +extern std::set hashAlgorithms; extern const std::string base32Chars; @@ -46,12 +46,12 @@ struct Hash size_t hashSize = 0; uint8_t hash[maxHashSize] = {}; - HashType type; + HashAlgorithm algo; /** * Create a zero-filled hash object. */ - explicit Hash(HashType type); + explicit Hash(HashAlgorithm algo); /** * Parse the hash from a string representation in the format @@ -60,7 +60,7 @@ struct Hash * is not present, then the hash type must be specified in the * string. */ - static Hash parseAny(std::string_view s, std::optional type); + static Hash parseAny(std::string_view s, std::optional optAlgo); /** * Parse a hash from a string representation like the above, except the @@ -72,7 +72,7 @@ struct Hash * Parse a plain hash that musst not have any prefix indicating the type. * The type is passed in to disambiguate. */ - static Hash parseNonSRIUnprefixed(std::string_view s, HashType type); + static Hash parseNonSRIUnprefixed(std::string_view s, HashAlgorithm algo); static Hash parseSRI(std::string_view original); @@ -81,7 +81,7 @@ private: * The type must be provided, the string view must not include * prefix. `isSRI` helps disambigate the various base-* encodings. */ - Hash(std::string_view s, HashType type, bool isSRI); + Hash(std::string_view s, HashAlgorithm algo, bool isSRI); public: /** @@ -125,10 +125,10 @@ public: /** * Return a string representation of the hash, in base-16, base-32 - * or base-64. By default, this is prefixed by the hash type + * or base-64. By default, this is prefixed by the hash algo * (e.g. "sha256:"). */ - [[nodiscard]] std::string to_string(HashFormat hashFormat, bool includeType) const; + [[nodiscard]] std::string to_string(HashFormat hashFormat, bool includeAlgo) const; [[nodiscard]] std::string gitRev() const { @@ -146,7 +146,7 @@ public: /** * Helper that defaults empty hashes to the 0 hash. */ -Hash newHashAllowEmpty(std::string_view hashStr, std::optional ht); +Hash newHashAllowEmpty(std::string_view hashStr, std::optional ha); /** * Print a hash in base-16 if it's MD5, or base-32 otherwise. @@ -156,14 +156,14 @@ std::string printHash16or32(const Hash & hash); /** * Compute the hash of the given string. */ -Hash hashString(HashType ht, std::string_view s); +Hash hashString(HashAlgorithm ha, std::string_view s); /** * Compute the hash of the given file, hashing its contents directly. * * (Metadata, such as the executable permission bit, is ignored.) */ -Hash hashFile(HashType ht, const Path & path); +Hash hashFile(HashAlgorithm ha, const Path & path); /** * Compute the hash of the given path, serializing as a Nix Archive and @@ -172,8 +172,8 @@ Hash hashFile(HashType ht, const Path & path); * The hash is defined as (essentially) hashString(ht, dumpPath(path)). */ typedef std::pair HashResult; -HashResult hashPath(HashType ht, const Path & path, - PathFilter & filter = defaultPathFilter); +HashResult hashPath(HashAlgorithm ha, const Path & path, + PathFilter & filter = defaultPathFilter); /** * Compress a hash to the specified number of bytes by cyclically @@ -199,17 +199,17 @@ std::string_view printHashFormat(HashFormat hashFormat); /** * Parse a string representing a hash type. */ -HashType parseHashType(std::string_view s); +HashAlgorithm parseHashAlgo(std::string_view s); /** * Will return nothing on parse error */ -std::optional parseHashTypeOpt(std::string_view s); +std::optional parseHashAlgoOpt(std::string_view s); /** * And the reverse. */ -std::string_view printHashType(HashType ht); +std::string_view printHashAlgo(HashAlgorithm ha); union Ctx; @@ -222,12 +222,12 @@ struct AbstractHashSink : virtual Sink class HashSink : public BufferedSink, public AbstractHashSink { private: - HashType ht; + HashAlgorithm ha; Ctx * ctx; uint64_t bytes; public: - HashSink(HashType ht); + HashSink(HashAlgorithm ha); HashSink(const HashSink & h); ~HashSink(); void writeUnbuffered(std::string_view data) override; diff --git a/src/libutil/references.cc b/src/libutil/references.cc index 9d75606ef..d82d51945 100644 --- a/src/libutil/references.cc +++ b/src/libutil/references.cc @@ -110,8 +110,8 @@ void RewritingSink::flush() prev.clear(); } -HashModuloSink::HashModuloSink(HashType ht, const std::string & modulus) - : hashSink(ht) +HashModuloSink::HashModuloSink(HashAlgorithm ha, const std::string & modulus) + : hashSink(ha) , rewritingSink(modulus, std::string(modulus.size(), 0), hashSink) { } diff --git a/src/libutil/references.hh b/src/libutil/references.hh index f0baeffe1..8bc9f7ec9 100644 --- a/src/libutil/references.hh +++ b/src/libutil/references.hh @@ -46,7 +46,7 @@ struct HashModuloSink : AbstractHashSink HashSink hashSink; RewritingSink rewritingSink; - HashModuloSink(HashType ht, const std::string & modulus); + HashModuloSink(HashAlgorithm ha, const std::string & modulus); void operator () (std::string_view data) override; diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index 7813433a7..afbbbe1a9 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -39,11 +39,11 @@ void SourceAccessor::readFile( } Hash SourceAccessor::hashPath( - const CanonPath & path, - PathFilter & filter, - HashType ht) + const CanonPath & path, + PathFilter & filter, + HashAlgorithm ha) { - HashSink sink(ht); + HashSink sink(ha); dumpPath(path, sink, filter); return sink.finish().first; } diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index 264caab16..3ca12d624 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -97,9 +97,9 @@ struct SourceAccessor PathFilter & filter = defaultPathFilter); Hash hashPath( - const CanonPath & path, - PathFilter & filter = defaultPathFilter, - HashType ht = htSHA256); + const CanonPath & path, + PathFilter & filter = defaultPathFilter, + HashAlgorithm ha = HashAlgorithm::SHA256); /** * Return a corresponding path in the root filesystem, if diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 25f0107bc..75ad4e75f 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -193,7 +193,7 @@ static void opAddFixed(Strings opFlags, Strings opArgs) if (opArgs.empty()) throw UsageError("first argument must be hash algorithm"); - HashType hashAlgo = parseHashType(opArgs.front()); + HashAlgorithm hashAlgo = parseHashAlgo(opArgs.front()); opArgs.pop_front(); for (auto & i : opArgs) @@ -214,7 +214,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) throw UsageError("'--print-fixed-path' requires three arguments"); Strings::iterator i = opArgs.begin(); - HashType hashAlgo = parseHashType(*i++); + HashAlgorithm hashAlgo = parseHashAlgo(*i++); std::string hash = *i++; std::string name = *i++; @@ -405,7 +405,7 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & j : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) { auto info = store->queryPathInfo(j); if (query == qHash) { - assert(info->narHash.type == htSHA256); + assert(info->narHash.algo == HashAlgorithm::SHA256); cout << fmt("%s\n", info->narHash.to_string(HashFormat::Base32, true)); } else if (query == qSize) cout << fmt("%d\n", info->narSize); @@ -541,7 +541,7 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) if (canonicalise) canonicalisePathMetaData(store->printStorePath(info->path), {}); if (!hashGiven) { - HashResult hash = hashPath(htSHA256, store->printStorePath(info->path)); + HashResult hash = hashPath(HashAlgorithm::SHA256, store->printStorePath(info->path)); info->narHash = hash.first; info->narSize = hash.second; } @@ -763,7 +763,7 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) auto path = store->followLinksToStorePath(i); printMsg(lvlTalkative, "checking path '%s'...", store->printStorePath(path)); auto info = store->queryPathInfo(path); - HashSink sink(info->narHash.type); + HashSink sink(info->narHash.algo); store->narFromPath(path, sink); auto current = sink.finish(); if (current.first != info->narHash) { @@ -979,7 +979,7 @@ static void opServe(Strings opFlags, Strings opArgs) auto deriver = readString(in); ValidPathInfo info { store->parseStorePath(path), - Hash::parseAny(readString(in), htSHA256), + Hash::parseAny(readString(in), HashAlgorithm::SHA256), }; if (deriver != "") info.deriver = store->parseStorePath(deriver); diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index f9d487ada..02de796b5 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -60,11 +60,11 @@ struct CmdAddToStore : MixDryRun, StoreCommand StringSink sink; dumpPath(path, sink); - auto narHash = hashString(htSHA256, sink.s); + auto narHash = hashString(HashAlgorithm::SHA256, sink.s); Hash hash = narHash; if (ingestionMethod == FileIngestionMethod::Flat) { - HashSink hsink(htSHA256); + HashSink hsink(HashAlgorithm::SHA256); readFile(path, hsink); hash = hsink.finish().first; } diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 62f96ef1d..638178afa 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -18,7 +18,7 @@ struct CmdHashBase : Command FileIngestionMethod mode; HashFormat hashFormat = HashFormat::SRI; bool truncate = false; - HashType ht = htSHA256; + HashAlgorithm ha = HashAlgorithm::SHA256; std::vector paths; std::optional modulus; @@ -48,7 +48,7 @@ struct CmdHashBase : Command .handler = {&hashFormat, HashFormat::Base16}, }); - addFlag(Flag::mkHashTypeFlag("type", &ht)); + addFlag(Flag::mkHashTypeFlag("type", &ha)); #if 0 addFlag({ @@ -84,9 +84,9 @@ struct CmdHashBase : Command std::unique_ptr hashSink; if (modulus) - hashSink = std::make_unique(ht, *modulus); + hashSink = std::make_unique(ha, *modulus); else - hashSink = std::make_unique(ht); + hashSink = std::make_unique(ha); switch (mode) { case FileIngestionMethod::Flat: @@ -107,7 +107,7 @@ struct CmdHashBase : Command struct CmdToBase : Command { HashFormat hashFormat; - std::optional ht; + std::optional ht; std::vector args; CmdToBase(HashFormat hashFormat) : hashFormat(hashFormat) @@ -139,7 +139,7 @@ struct CmdHashConvert : Command { std::optional from; HashFormat to; - std::optional type; + std::optional type; std::vector hashStrings; CmdHashConvert(): to(HashFormat::SRI) { @@ -166,7 +166,7 @@ struct CmdHashConvert : Command .description = "Specify the algorithm if it can't be auto-detected.", .labels = {"hash algorithm"}, .handler = {[this](std::string str) { - type = parseHashType(str); + type = parseHashAlgo(str); }}, }); expectArgs({ @@ -223,7 +223,7 @@ static auto rCmdHash = registerCommand("hash"); /* Legacy nix-hash command. */ static int compatNixHash(int argc, char * * argv) { - std::optional ht; + std::optional ha; bool flat = false; HashFormat hashFormat = HashFormat::Base16; bool truncate = false; @@ -243,7 +243,7 @@ static int compatNixHash(int argc, char * * argv) else if (*arg == "--truncate") truncate = true; else if (*arg == "--type") { std::string s = getArg(*arg, arg, end); - ht = parseHashType(s); + ha = parseHashAlgo(s); } else if (*arg == "--to-base16") { op = opTo; @@ -270,8 +270,8 @@ static int compatNixHash(int argc, char * * argv) if (op == opHash) { CmdHashBase cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive); - if (!ht.has_value()) ht = htMD5; - cmd.ht = ht.value(); + if (!ha.has_value()) ha = HashAlgorithm::MD5; + cmd.ha = ha.value(); cmd.hashFormat = hashFormat; cmd.truncate = truncate; cmd.paths = ss; @@ -281,7 +281,7 @@ static int compatNixHash(int argc, char * * argv) else { CmdToBase cmd(hashFormat); cmd.args = ss; - if (ht.has_value()) cmd.ht = ht; + if (ha.has_value()) cmd.ht = ha; cmd.run(); } diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 3ed7946a8..09f33a51e 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -46,13 +46,13 @@ std::string resolveMirrorUrl(EvalState & state, const std::string & url) } std::tuple prefetchFile( - ref store, - std::string_view url, - std::optional name, - HashType hashType, - std::optional expectedHash, - bool unpack, - bool executable) + ref store, + std::string_view url, + std::optional name, + HashAlgorithm hashAlgo, + std::optional expectedHash, + bool unpack, + bool executable) { auto ingestionMethod = unpack || executable ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; @@ -69,7 +69,7 @@ std::tuple prefetchFile( /* If an expected hash is given, the file may already exist in the store. */ if (expectedHash) { - hashType = expectedHash->type; + hashAlgo = expectedHash->algo; storePath = store->makeFixedOutputPath(*name, FixedOutputInfo { .method = ingestionMethod, .hash = *expectedHash, @@ -122,7 +122,7 @@ std::tuple prefetchFile( Activity act(*logger, lvlChatty, actUnknown, fmt("adding '%s' to the store", url)); - auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash); + auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashAlgo, expectedHash); storePath = info.path; assert(info.ca); hash = info.ca->hash; @@ -134,7 +134,7 @@ std::tuple prefetchFile( static int main_nix_prefetch_url(int argc, char * * argv) { { - HashType ht = htSHA256; + HashAlgorithm ha = HashAlgorithm::SHA256; std::vector args; bool printPath = getEnv("PRINT_PATH") == "1"; bool fromExpr = false; @@ -155,7 +155,7 @@ static int main_nix_prefetch_url(int argc, char * * argv) printVersion("nix-prefetch-url"); else if (*arg == "--type") { auto s = getArg(*arg, arg, end); - ht = parseHashType(s); + ha = parseHashAlgo(s); } else if (*arg == "--print-path") printPath = true; @@ -233,10 +233,10 @@ static int main_nix_prefetch_url(int argc, char * * argv) std::optional expectedHash; if (args.size() == 2) - expectedHash = Hash::parseAny(args[1], ht); + expectedHash = Hash::parseAny(args[1], ha); auto [storePath, hash] = prefetchFile( - store, resolveMirrorUrl(*state, url), name, ht, expectedHash, unpack, executable); + store, resolveMirrorUrl(*state, url), name, ha, expectedHash, unpack, executable); stopProgressBar(); @@ -258,7 +258,7 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON std::string url; bool executable = false; std::optional name; - HashType hashType = htSHA256; + HashAlgorithm hashAlgo = HashAlgorithm::SHA256; std::optional expectedHash; CmdStorePrefetchFile() @@ -275,11 +275,11 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON .description = "The expected hash of the file.", .labels = {"hash"}, .handler = {[&](std::string s) { - expectedHash = Hash::parseAny(s, hashType); + expectedHash = Hash::parseAny(s, hashAlgo); }} }); - addFlag(Flag::mkHashTypeFlag("hash-type", &hashType)); + addFlag(Flag::mkHashTypeFlag("hash-type", &hashAlgo)); addFlag({ .longName = "executable", @@ -305,7 +305,7 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON } void run(ref store) override { - auto [storePath, hash] = prefetchFile(store, url, name, hashType, expectedHash, false, executable); + auto [storePath, hash] = prefetchFile(store, url, name, hashAlgo, expectedHash, false, executable); if (json) { auto res = nlohmann::json::object(); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 147b4680b..9d9492da9 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -216,7 +216,7 @@ struct ProfileManifest StringSink sink; dumpPath(tempDir, sink); - auto narHash = hashString(htSHA256, sink.s); + auto narHash = hashString(HashAlgorithm::SHA256, sink.s); ValidPathInfo info { *store, diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 78cb765ce..cd0f6d95f 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -98,7 +98,7 @@ struct CmdVerify : StorePathsCommand if (!noContents) { - auto hashSink = HashSink(info->narHash.type); + auto hashSink = HashSink(info->narHash.algo); store->narFromPath(info->path, hashSink); diff --git a/tests/unit/libstore/common-protocol.cc b/tests/unit/libstore/common-protocol.cc index c09ac6a3e..d23805fc3 100644 --- a/tests/unit/libstore/common-protocol.cc +++ b/tests/unit/libstore/common-protocol.cc @@ -84,15 +84,15 @@ CHARACTERIZATION_TEST( (std::tuple { ContentAddress { .method = TextIngestionMethod {}, - .hash = hashString(HashType::htSHA256, "Derive(...)"), + .hash = hashString(HashAlgorithm::SHA256, "Derive(...)"), }, ContentAddress { .method = FileIngestionMethod::Flat, - .hash = hashString(HashType::htSHA1, "blob blob..."), + .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, ContentAddress { .method = FileIngestionMethod::Recursive, - .hash = hashString(HashType::htSHA256, "(...)"), + .hash = hashString(HashAlgorithm::SHA256, "(...)"), }, })) @@ -179,7 +179,7 @@ CHARACTERIZATION_TEST( std::optional { ContentAddress { .method = FileIngestionMethod::Flat, - .hash = hashString(HashType::htSHA1, "blob blob..."), + .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, }, })) diff --git a/tests/unit/libstore/derivation.cc b/tests/unit/libstore/derivation.cc index a7f4488fa..7a4b1403a 100644 --- a/tests/unit/libstore/derivation.cc +++ b/tests/unit/libstore/derivation.cc @@ -134,7 +134,7 @@ TEST_JSON(DynDerivationTest, caFixedText, TEST_JSON(CaDerivationTest, caFloating, (DerivationOutput::CAFloating { .method = FileIngestionMethod::Recursive, - .hashType = htSHA256, + .hashAlgo = HashAlgorithm::SHA256, }), "drv-name", "output-name") @@ -145,7 +145,7 @@ TEST_JSON(DerivationTest, deferred, TEST_JSON(ImpureDerivationTest, impure, (DerivationOutput::Impure { .method = FileIngestionMethod::Recursive, - .hashType = htSHA256, + .hashAlgo = HashAlgorithm::SHA256, }), "drv-name", "output-name") diff --git a/tests/unit/libstore/nar-info.cc b/tests/unit/libstore/nar-info.cc index 4f124e89e..bd10602e7 100644 --- a/tests/unit/libstore/nar-info.cc +++ b/tests/unit/libstore/nar-info.cc @@ -26,7 +26,7 @@ static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo) { "foo", FixedOutputInfo { .method = FileIngestionMethod::Recursive, - .hash = hashString(HashType::htSHA256, "(...)"), + .hash = hashString(HashAlgorithm::SHA256, "(...)"), .references = { .others = { diff --git a/tests/unit/libstore/path-info.cc b/tests/unit/libstore/path-info.cc index 18f00ca19..80d6fcfed 100644 --- a/tests/unit/libstore/path-info.cc +++ b/tests/unit/libstore/path-info.cc @@ -25,7 +25,7 @@ static UnkeyedValidPathInfo makePathInfo(const Store & store, bool includeImpure "foo", FixedOutputInfo { .method = FileIngestionMethod::Recursive, - .hash = hashString(HashType::htSHA256, "(...)"), + .hash = hashString(HashAlgorithm::SHA256, "(...)"), .references = { .others = { diff --git a/tests/unit/libstore/serve-protocol.cc b/tests/unit/libstore/serve-protocol.cc index c8ac87a04..6d2054f7d 100644 --- a/tests/unit/libstore/serve-protocol.cc +++ b/tests/unit/libstore/serve-protocol.cc @@ -53,15 +53,15 @@ VERSIONED_CHARACTERIZATION_TEST( (std::tuple { ContentAddress { .method = TextIngestionMethod {}, - .hash = hashString(HashType::htSHA256, "Derive(...)"), + .hash = hashString(HashAlgorithm::SHA256, "Derive(...)"), }, ContentAddress { .method = FileIngestionMethod::Flat, - .hash = hashString(HashType::htSHA1, "blob blob..."), + .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, ContentAddress { .method = FileIngestionMethod::Recursive, - .hash = hashString(HashType::htSHA256, "(...)"), + .hash = hashString(HashAlgorithm::SHA256, "(...)"), }, })) @@ -271,7 +271,7 @@ VERSIONED_CHARACTERIZATION_TEST( std::optional { ContentAddress { .method = FileIngestionMethod::Flat, - .hash = hashString(HashType::htSHA1, "blob blob..."), + .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, }, })) diff --git a/tests/unit/libstore/worker-protocol.cc b/tests/unit/libstore/worker-protocol.cc index ad5943c69..91f804f0c 100644 --- a/tests/unit/libstore/worker-protocol.cc +++ b/tests/unit/libstore/worker-protocol.cc @@ -55,15 +55,15 @@ VERSIONED_CHARACTERIZATION_TEST( (std::tuple { ContentAddress { .method = TextIngestionMethod {}, - .hash = hashString(HashType::htSHA256, "Derive(...)"), + .hash = hashString(HashAlgorithm::SHA256, "Derive(...)"), }, ContentAddress { .method = FileIngestionMethod::Flat, - .hash = hashString(HashType::htSHA1, "blob blob..."), + .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, ContentAddress { .method = FileIngestionMethod::Recursive, - .hash = hashString(HashType::htSHA256, "(...)"), + .hash = hashString(HashAlgorithm::SHA256, "(...)"), }, })) @@ -464,7 +464,7 @@ VERSIONED_CHARACTERIZATION_TEST( "foo", FixedOutputInfo { .method = FileIngestionMethod::Recursive, - .hash = hashString(HashType::htSHA256, "(...)"), + .hash = hashString(HashAlgorithm::SHA256, "(...)"), .references = { .others = { StorePath { @@ -539,7 +539,7 @@ VERSIONED_CHARACTERIZATION_TEST( std::optional { ContentAddress { .method = FileIngestionMethod::Flat, - .hash = hashString(HashType::htSHA1, "blob blob..."), + .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, }, })) diff --git a/tests/unit/libutil/git.cc b/tests/unit/libutil/git.cc index 551a2d105..141a55816 100644 --- a/tests/unit/libutil/git.cc +++ b/tests/unit/libutil/git.cc @@ -95,7 +95,7 @@ const static Tree tree = { { .mode = Mode::Regular, // hello world with special chars from above - .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", htSHA1), + .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", HashAlgorithm::SHA1), }, }, { @@ -103,7 +103,7 @@ const static Tree tree = { { .mode = Mode::Executable, // ditto - .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", htSHA1), + .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", HashAlgorithm::SHA1), }, }, { @@ -111,7 +111,7 @@ const static Tree tree = { { .mode = Mode::Directory, // Empty directory hash - .hash = Hash::parseAny("4b825dc642cb6eb9a060e54bf8d69288fbee4904", htSHA1), + .hash = Hash::parseAny("4b825dc642cb6eb9a060e54bf8d69288fbee4904", HashAlgorithm::SHA1), }, }, }; @@ -174,7 +174,7 @@ TEST_F(GitTest, both_roundrip) { std::function dumpHook; dumpHook = [&](const CanonPath & path) { StringSink s; - HashSink hashSink { htSHA1 }; + HashSink hashSink { HashAlgorithm::SHA1 }; TeeSink s2 { s, hashSink }; auto mode = dump( files, path, s2, dumpHook, diff --git a/tests/unit/libutil/hash.cc b/tests/unit/libutil/hash.cc index 92291afce..4d82c7f09 100644 --- a/tests/unit/libutil/hash.cc +++ b/tests/unit/libutil/hash.cc @@ -13,28 +13,28 @@ namespace nix { TEST(hashString, testKnownMD5Hashes1) { // values taken from: https://tools.ietf.org/html/rfc1321 auto s1 = ""; - auto hash = hashString(HashType::htMD5, s1); + auto hash = hashString(HashAlgorithm::MD5, s1); ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "md5:d41d8cd98f00b204e9800998ecf8427e"); } TEST(hashString, testKnownMD5Hashes2) { // values taken from: https://tools.ietf.org/html/rfc1321 auto s2 = "abc"; - auto hash = hashString(HashType::htMD5, s2); + auto hash = hashString(HashAlgorithm::MD5, s2); ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "md5:900150983cd24fb0d6963f7d28e17f72"); } TEST(hashString, testKnownSHA1Hashes1) { // values taken from: https://tools.ietf.org/html/rfc3174 auto s = "abc"; - auto hash = hashString(HashType::htSHA1, s); + auto hash = hashString(HashAlgorithm::SHA1, s); ASSERT_EQ(hash.to_string(HashFormat::Base16, true),"sha1:a9993e364706816aba3e25717850c26c9cd0d89d"); } TEST(hashString, testKnownSHA1Hashes2) { // values taken from: https://tools.ietf.org/html/rfc3174 auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; - auto hash = hashString(HashType::htSHA1, s); + auto hash = hashString(HashAlgorithm::SHA1, s); ASSERT_EQ(hash.to_string(HashFormat::Base16, true),"sha1:84983e441c3bd26ebaae4aa1f95129e5e54670f1"); } @@ -42,7 +42,7 @@ namespace nix { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abc"; - auto hash = hashString(HashType::htSHA256, s); + auto hash = hashString(HashAlgorithm::SHA256, s); ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); } @@ -50,7 +50,7 @@ namespace nix { TEST(hashString, testKnownSHA256Hashes2) { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; - auto hash = hashString(HashType::htSHA256, s); + auto hash = hashString(HashAlgorithm::SHA256, s); ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "sha256:248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1"); } @@ -58,7 +58,7 @@ namespace nix { TEST(hashString, testKnownSHA512Hashes1) { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abc"; - auto hash = hashString(HashType::htSHA512, s); + auto hash = hashString(HashAlgorithm::SHA512, s); ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "sha512:ddaf35a193617abacc417349ae20413112e6fa4e89a9" "7ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd" @@ -68,7 +68,7 @@ namespace nix { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"; - auto hash = hashString(HashType::htSHA512, s); + auto hash = hashString(HashAlgorithm::SHA512, s); ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "sha512:8e959b75dae313da8cf4f72814fc143f8f7779c6eb9f7fa1" "7299aeadb6889018501d289e4900f7e4331b99dec4b5433a" From 837b889c41543b32154ceade2363ec6ad6dff15d Mon Sep 17 00:00:00 2001 From: Peter Kolloch Date: Tue, 28 Nov 2023 14:41:44 +0100 Subject: [PATCH 082/141] Further HashType renaming + using mkHashAlgoOptFlag for new conversion https://github.com/NixOS/nix/issues/8876 --- src/libutil/args.cc | 38 +++++++++++++++++++------------------- src/libutil/args.hh | 4 ++-- src/nix/hash.cc | 17 +++++------------ src/nix/prefetch.cc | 2 +- 4 files changed, 27 insertions(+), 34 deletions(-) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index ac3727d11..7ea1647d9 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -544,36 +544,36 @@ nlohmann::json Args::toJSON() return res; } -static void hashTypeCompleter(AddCompletions & completions, size_t index, std::string_view prefix) +static void hashAlgoCompleter(AddCompletions & completions, size_t index, std::string_view prefix) { for (auto & type : hashAlgorithms) if (hasPrefix(type, prefix)) completions.add(type); } -Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashAlgorithm * ha) +Args::Flag Args::Flag::mkHashAlgoFlag(std::string && longName, HashAlgorithm * ha) { - return Flag { - .longName = std::move(longName), - .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')", - .labels = {"hash-algo"}, - .handler = {[ha](std::string s) { - *ha = parseHashAlgo(s); - }}, - .completer = hashTypeCompleter, + return Flag{ + .longName = std::move(longName), + .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')", + .labels = {"hash-algo"}, + .handler = {[ha](std::string s) { + *ha = parseHashAlgo(s); + }}, + .completer = hashAlgoCompleter, }; } -Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional * oha) +Args::Flag Args::Flag::mkHashAlgoOptFlag(std::string && longName, std::optional * oha) { - return Flag { - .longName = std::move(longName), - .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512'). Optional as can also be gotten from SRI hash itself.", - .labels = {"hash-algo"}, - .handler = {[oha](std::string s) { - *oha = std::optional {parseHashAlgo(s) }; - }}, - .completer = hashTypeCompleter, + return Flag{ + .longName = std::move(longName), + .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512'). Optional as can also be gotten from SRI hash itself.", + .labels = {"hash-algo"}, + .handler = {[oha](std::string s) { + *oha = std::optional{parseHashAlgo(s)}; + }}, + .completer = hashAlgoCompleter, }; } diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 0cff76158..653a9bbd6 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -175,8 +175,8 @@ protected: std::optional experimentalFeature; - static Flag mkHashTypeFlag(std::string && longName, HashAlgorithm * ha); - static Flag mkHashTypeOptFlag(std::string && longName, std::optional * oha); + static Flag mkHashAlgoFlag(std::string && longName, HashAlgorithm * ha); + static Flag mkHashAlgoOptFlag(std::string && longName, std::optional * oha); }; /** diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 638178afa..173043c8a 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -48,7 +48,7 @@ struct CmdHashBase : Command .handler = {&hashFormat, HashFormat::Base16}, }); - addFlag(Flag::mkHashTypeFlag("type", &ha)); + addFlag(Flag::mkHashAlgoFlag("type", &ha)); #if 0 addFlag({ @@ -112,7 +112,7 @@ struct CmdToBase : Command CmdToBase(HashFormat hashFormat) : hashFormat(hashFormat) { - addFlag(Flag::mkHashTypeOptFlag("type", &ht)); + addFlag(Flag::mkHashAlgoOptFlag("type", &ht)); expectArgs("strings", &args); } @@ -139,7 +139,7 @@ struct CmdHashConvert : Command { std::optional from; HashFormat to; - std::optional type; + std::optional algo; std::vector hashStrings; CmdHashConvert(): to(HashFormat::SRI) { @@ -161,14 +161,7 @@ struct CmdHashConvert : Command to = parseHashFormat(str); }}, }); - addFlag({ - .longName = "algo", - .description = "Specify the algorithm if it can't be auto-detected.", - .labels = {"hash algorithm"}, - .handler = {[this](std::string str) { - type = parseHashAlgo(str); - }}, - }); + addFlag(Args::Flag::mkHashAlgoOptFlag("algo", &algo)); expectArgs({ .label = "hashes", .handler = {&hashStrings}, @@ -184,7 +177,7 @@ struct CmdHashConvert : Command void run() override { for (const auto& s: hashStrings) { - Hash h = Hash::parseAny(s, type); + Hash h = Hash::parseAny(s, algo); if (from && h.to_string(*from, from == HashFormat::SRI) != s) { auto from_as_string = printHashFormat(*from); throw BadHash("input hash '%s' does not have the expected format '--from %s'", s, from_as_string); diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 09f33a51e..bbfeb8aa4 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -279,7 +279,7 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON }} }); - addFlag(Flag::mkHashTypeFlag("hash-type", &hashAlgo)); + addFlag(Flag::mkHashAlgoFlag("hash-type", &hashAlgo)); addFlag({ .longName = "executable", From fc6f29053aa69b6b14bcad93cb273b1c266e74fe Mon Sep 17 00:00:00 2001 From: Peter Kolloch Date: Tue, 28 Nov 2023 15:38:15 +0100 Subject: [PATCH 083/141] Renamed HashFormat::Base32 to HashFormat::Nix32 ...and also adjusted parsing accordingly. Also added CLI completion for HashFormats. https://github.com/NixOS/nix/issues/8876 --- src/libexpr/primops/fetchTree.cc | 2 +- src/libfetchers/git.cc | 2 +- src/libfetchers/mercurial.cc | 2 +- src/libstore/binary-cache-store.cc | 4 +- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/content-address.cc | 2 +- src/libstore/derivations.cc | 2 +- src/libstore/downstream-placeholder.cc | 4 +- src/libstore/export-import.cc | 2 +- src/libstore/gc.cc | 2 +- src/libstore/local-store.cc | 10 +- src/libstore/nar-info-disk-cache.cc | 4 +- src/libstore/nar-info.cc | 4 +- src/libstore/optimise-store.cc | 4 +- src/libstore/parsed-derivations.cc | 2 +- src/libstore/path-info.cc | 6 +- src/libstore/path.cc | 2 +- src/libutil/args.cc | 40 ++++++- src/libutil/args.hh | 3 + src/libutil/hash.cc | 25 ++-- src/libutil/hash.hh | 10 +- src/libutil/references.cc | 4 +- src/nix-store/nix-store.cc | 8 +- src/nix/hash.cc | 40 +++---- src/nix/verify.cc | 4 +- tests/functional/hash.sh | 6 +- .../lang/eval-okay-convertHash.err.exp | 108 ++++++++++++++++++ .../functional/lang/eval-okay-convertHash.exp | 2 +- .../functional/lang/eval-okay-convertHash.nix | 2 + tests/unit/libutil/hash.cc | 2 +- 30 files changed, 228 insertions(+), 82 deletions(-) create mode 100644 tests/functional/lang/eval-okay-convertHash.err.exp diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index ef80c634f..15f870a95 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -304,7 +304,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v : hashFile(HashAlgorithm::SHA256, state.store->toRealPath(storePath)); if (hash != *expectedHash) state.debugThrowLastTrace(EvalError((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s", - *url, expectedHash->to_string(HashFormat::Base32, true), hash.to_string(HashFormat::Base32, true))); + *url, expectedHash->to_string(HashFormat::Nix32, true), hash.to_string(HashFormat::Nix32, true))); } state.allowAndSetStorePathString(storePath, v); diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index a89acc1c0..9e6ba8963 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -52,7 +52,7 @@ bool touchCacheFile(const Path & path, time_t touch_time) Path getCachePath(std::string_view key) { return getCacheDir() + "/nix/gitv3/" + - hashString(HashAlgorithm::SHA256, key).to_string(HashFormat::Base32, false); + hashString(HashAlgorithm::SHA256, key).to_string(HashFormat::Nix32, false); } // Returns the name of the HEAD branch. diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 713f24bbb..6056b9a3c 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -267,7 +267,7 @@ struct MercurialInputScheme : InputScheme } } - Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(HashAlgorithm::SHA256, actualUrl).to_string(HashFormat::Base32, false)); + Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(HashAlgorithm::SHA256, actualUrl).to_string(HashFormat::Nix32, false)); /* If this is a commit hash that we already have, we don't have to pull again. */ diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index f287d72a8..2837e8934 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -165,8 +165,8 @@ ref BinaryCacheStore::addToStoreCommon( auto [fileHash, fileSize] = fileHashSink.finish(); narInfo->fileHash = fileHash; narInfo->fileSize = fileSize; - narInfo->url = "nar/" + narInfo->fileHash->to_string(HashFormat::Base32, false) + ".nar" - + (compression == "xz" ? ".xz" : + narInfo->url = "nar/" + narInfo->fileHash->to_string(HashFormat::Nix32, false) + ".nar" + + (compression == "xz" ? ".xz" : compression == "bzip2" ? ".bz2" : compression == "zstd" ? ".zst" : compression == "lzip" ? ".lzip" : diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 4c3dc1f5c..802b39f84 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1067,7 +1067,7 @@ void LocalDerivationGoal::initTmpDir() { env[i.first] = i.second; } else { auto hash = hashString(HashAlgorithm::SHA256, i.first); - std::string fn = ".attr-" + hash.to_string(HashFormat::Base32, false); + std::string fn = ".attr-" + hash.to_string(HashFormat::Nix32, false); Path p = tmpDir + "/" + fn; writeFile(p, rewriteStrings(i.second, inputRewrites)); chownToBuilder(p); diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index de8194f73..f42a13126 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -61,7 +61,7 @@ std::string ContentAddress::render() const + makeFileIngestionPrefix(method); }, }, method.raw) - + this->hash.to_string(HashFormat::Base32, true); + + this->hash.to_string(HashFormat::Nix32, true); } /** diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index c68631c1a..664ab7556 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -958,7 +958,7 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva std::string hashPlaceholder(const OutputNameView outputName) { // FIXME: memoize? - return "/" + hashString(HashAlgorithm::SHA256, concatStrings("nix-output:", outputName)).to_string(HashFormat::Base32, false); + return "/" + hashString(HashAlgorithm::SHA256, concatStrings("nix-output:", outputName)).to_string(HashFormat::Nix32, false); } diff --git a/src/libstore/downstream-placeholder.cc b/src/libstore/downstream-placeholder.cc index 10df37fa4..91d47f946 100644 --- a/src/libstore/downstream-placeholder.cc +++ b/src/libstore/downstream-placeholder.cc @@ -5,7 +5,7 @@ namespace nix { std::string DownstreamPlaceholder::render() const { - return "/" + hash.to_string(HashFormat::Base32, false); + return "/" + hash.to_string(HashFormat::Nix32, false); } @@ -31,7 +31,7 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation( xpSettings.require(Xp::DynamicDerivations); auto compressed = compressHash(placeholder.hash, 20); auto clearText = "nix-computed-output:" - + compressed.to_string(HashFormat::Base32, false) + + compressed.to_string(HashFormat::Nix32, false) + ":" + std::string { outputName }; return DownstreamPlaceholder { hashString(HashAlgorithm::SHA256, clearText) diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 48718ef84..d57b25bd7 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -41,7 +41,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) Hash hash = hashSink.currentHash().first; if (hash != info->narHash && info->narHash != Hash(info->narHash.algo)) throw Error("hash of path '%s' has changed from '%s' to '%s'!", - printStorePath(path), info->narHash.to_string(HashFormat::Base32, true), hash.to_string(HashFormat::Base32, true)); + printStorePath(path), info->narHash.to_string(HashFormat::Nix32, true), hash.to_string(HashFormat::Nix32, true)); teeSink << exportMagic diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 5c413aa77..2bd3a2edc 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -50,7 +50,7 @@ static void makeSymlink(const Path & link, const Path & target) void LocalStore::addIndirectRoot(const Path & path) { - std::string hash = hashString(HashAlgorithm::SHA1, path).to_string(HashFormat::Base32, false); + std::string hash = hashString(HashAlgorithm::SHA1, path).to_string(HashFormat::Nix32, false); Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", stateDir, gcRootsDir, hash)); makeSymlink(realRoot, path); } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index ef7dd7985..7e82bae28 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1080,7 +1080,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (hashResult.first != info.narHash) throw Error("hash mismatch importing path '%s';\n specified: %s\n got: %s", - printStorePath(info.path), info.narHash.to_string(HashFormat::Base32, true), hashResult.first.to_string(HashFormat::Base32, true)); + printStorePath(info.path), info.narHash.to_string(HashFormat::Nix32, true), hashResult.first.to_string(HashFormat::Nix32, true)); if (hashResult.second != info.narSize) throw Error("size mismatch importing path '%s';\n specified: %s\n got: %s", @@ -1096,8 +1096,8 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (specified.hash != actualHash.hash) { throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", printStorePath(info.path), - specified.hash.to_string(HashFormat::Base32, true), - actualHash.hash.to_string(HashFormat::Base32, true)); + specified.hash.to_string(HashFormat::Nix32, true), + actualHash.hash.to_string(HashFormat::Nix32, true)); } } @@ -1389,7 +1389,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) for (auto & link : readDirectory(linksDir)) { printMsg(lvlTalkative, "checking contents of '%s'", link.name); Path linkPath = linksDir + "/" + link.name; - std::string hash = hashPath(HashAlgorithm::SHA256, linkPath).first.to_string(HashFormat::Base32, false); + std::string hash = hashPath(HashAlgorithm::SHA256, linkPath).first.to_string(HashFormat::Nix32, false); if (hash != link.name) { printError("link '%s' was modified! expected hash '%s', got '%s'", linkPath, link.name, hash); @@ -1422,7 +1422,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) if (info->narHash != nullHash && info->narHash != current.first) { printError("path '%s' was modified! expected hash '%s', got '%s'", - printStorePath(i), info->narHash.to_string(HashFormat::Base32, true), current.first.to_string(HashFormat::Base32, true)); + printStorePath(i), info->narHash.to_string(HashFormat::Nix32, true), current.first.to_string(HashFormat::Nix32, true)); if (repair) repairPath(i); else errors = true; } else { diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index e50c15939..310105c75 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -333,9 +333,9 @@ public: (std::string(info->path.name())) (narInfo ? narInfo->url : "", narInfo != 0) (narInfo ? narInfo->compression : "", narInfo != 0) - (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(HashFormat::Base32, true) : "", narInfo && narInfo->fileHash) + (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(HashFormat::Nix32, true) : "", narInfo && narInfo->fileHash) (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize) - (info->narHash.to_string(HashFormat::Base32, true)) + (info->narHash.to_string(HashFormat::Nix32, true)) (info->narSize) (concatStringsSep(" ", info->shortRefs())) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 25e2a7d7b..d9618d04c 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -114,10 +114,10 @@ std::string NarInfo::to_string(const Store & store) const assert(compression != ""); res += "Compression: " + compression + "\n"; assert(fileHash && fileHash->algo == HashAlgorithm::SHA256); - res += "FileHash: " + fileHash->to_string(HashFormat::Base32, true) + "\n"; + res += "FileHash: " + fileHash->to_string(HashFormat::Nix32, true) + "\n"; res += "FileSize: " + std::to_string(fileSize) + "\n"; assert(narHash.algo == HashAlgorithm::SHA256); - res += "NarHash: " + narHash.to_string(HashFormat::Base32, true) + "\n"; + res += "NarHash: " + narHash.to_string(HashFormat::Nix32, true) + "\n"; res += "NarSize: " + std::to_string(narSize) + "\n"; res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index cadf88347..b395453d1 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -147,10 +147,10 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, contents of the symlink (i.e. the result of readlink()), not the contents of the target (which may not even exist). */ Hash hash = hashPath(HashAlgorithm::SHA256, path).first; - debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Base32, true)); + debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Nix32, true)); /* Check if this is a known hash. */ - Path linkPath = linksDir + "/" + hash.to_string(HashFormat::Base32, false); + Path linkPath = linksDir + "/" + hash.to_string(HashFormat::Nix32, false); /* Maybe delete the link, if it has been corrupted. */ if (pathExists(linkPath)) { diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index 73e55a96c..72f45143d 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -146,7 +146,7 @@ static nlohmann::json pathInfoToJSON( auto info = store.queryPathInfo(storePath); auto & jsonPath = jsonList.emplace_back( - info->toJSON(store, false, HashFormat::Base32)); + info->toJSON(store, false, HashFormat::Nix32)); // Add the path to the object whose metadata we are including. jsonPath["path"] = store.printStorePath(storePath); diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 2d7dc972f..f58e31bfd 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -31,9 +31,9 @@ std::string ValidPathInfo::fingerprint(const Store & store) const throw Error("cannot calculate fingerprint of path '%s' because its size is not known", store.printStorePath(path)); return - "1;" + store.printStorePath(path) + ";" - + narHash.to_string(HashFormat::Base32, true) + ";" - + std::to_string(narSize) + ";" + "1;" + store.printStorePath(path) + ";" + + narHash.to_string(HashFormat::Nix32, true) + ";" + + std::to_string(narSize) + ";" + concatStringsSep(",", store.printStorePathSet(references)); } diff --git a/src/libstore/path.cc b/src/libstore/path.cc index d5257c939..1afd10af7 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -35,7 +35,7 @@ StorePath::StorePath(std::string_view _baseName) } StorePath::StorePath(const Hash & hash, std::string_view _name) - : baseName((hash.to_string(HashFormat::Base32, false) + "-").append(std::string(_name))) + : baseName((hash.to_string(HashFormat::Nix32, false) + "-").append(std::string(_name))) { checkName(baseName, name()); } diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 7ea1647d9..e2668c673 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -544,11 +544,45 @@ nlohmann::json Args::toJSON() return res; } +static void hashFormatCompleter(AddCompletions & completions, size_t index, std::string_view prefix) +{ + for (auto & format : hashFormats) { + if (hasPrefix(format, prefix)) { + completions.add(format); + } + } +} + +Args::Flag Args::Flag::mkHashFormatFlagWithDefault(std::string &&longName, HashFormat * hf) { + assert(*hf == nix::HashFormat::SRI); + return Flag{ + .longName = std::move(longName), + .description = "hash format ('base16', 'nix32', 'base64', 'sri'). Default: 'sri'", + .labels = {"hash-format"}, + .handler = {[hf](std::string s) { + *hf = parseHashFormat(s); + }}, + .completer = hashFormatCompleter, + }; +} + +Args::Flag Args::Flag::mkHashFormatOptFlag(std::string && longName, std::optional * ohf) { + return Flag{ + .longName = std::move(longName), + .description = "hash format ('base16', 'nix32', 'base64', 'sri').", + .labels = {"hash-format"}, + .handler = {[ohf](std::string s) { + *ohf = std::optional{parseHashFormat(s)}; + }}, + .completer = hashFormatCompleter, + }; +} + static void hashAlgoCompleter(AddCompletions & completions, size_t index, std::string_view prefix) { - for (auto & type : hashAlgorithms) - if (hasPrefix(type, prefix)) - completions.add(type); + for (auto & algo : hashAlgorithms) + if (hasPrefix(algo, prefix)) + completions.add(algo); } Args::Flag Args::Flag::mkHashAlgoFlag(std::string && longName, HashAlgorithm * ha) diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 653a9bbd6..18b0ae583 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -15,6 +15,7 @@ namespace nix { enum struct HashAlgorithm : char; +enum struct HashFormat : int; class MultiCommand; @@ -177,6 +178,8 @@ protected: static Flag mkHashAlgoFlag(std::string && longName, HashAlgorithm * ha); static Flag mkHashAlgoOptFlag(std::string && longName, std::optional * oha); + static Flag mkHashFormatFlagWithDefault(std::string && longName, HashFormat * hf); + static Flag mkHashFormatOptFlag(std::string && longName, std::optional * ohf); }; /** diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 38a29c459..30456ae5c 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -27,8 +27,9 @@ static size_t regularHashSize(HashAlgorithm type) { } -std::set hashAlgorithms = {"md5", "sha1", "sha256", "sha512" }; +const std::set hashAlgorithms = {"md5", "sha1", "sha256", "sha512" }; +const std::set hashFormats = {"base64", "nix32", "base16", "sri" }; Hash::Hash(HashAlgorithm algo) : algo(algo) { @@ -81,7 +82,7 @@ static std::string printHash16(const Hash & hash) // omitted: E O U T -const std::string base32Chars = "0123456789abcdfghijklmnpqrsvwxyz"; +const std::string nix32Chars = "0123456789abcdfghijklmnpqrsvwxyz"; static std::string printHash32(const Hash & hash) @@ -100,7 +101,7 @@ static std::string printHash32(const Hash & hash) unsigned char c = (hash.hash[i] >> j) | (i >= hash.hashSize - 1 ? 0 : hash.hash[i + 1] << (8 - j)); - s.push_back(base32Chars[c & 0x1f]); + s.push_back(nix32Chars[c & 0x1f]); } return s; @@ -110,7 +111,7 @@ static std::string printHash32(const Hash & hash) std::string printHash16or32(const Hash & hash) { assert(static_cast(hash.algo)); - return hash.to_string(hash.algo == HashAlgorithm::MD5 ? HashFormat::Base16 : HashFormat::Base32, false); + return hash.to_string(hash.algo == HashAlgorithm::MD5 ? HashFormat::Base16 : HashFormat::Nix32, false); } @@ -125,7 +126,7 @@ std::string Hash::to_string(HashFormat hashFormat, bool includeAlgo) const case HashFormat::Base16: s += printHash16(*this); break; - case HashFormat::Base32: + case HashFormat::Nix32: s += printHash32(*this); break; case HashFormat::Base64: @@ -230,8 +231,8 @@ Hash::Hash(std::string_view rest, HashAlgorithm algo, bool isSRI) for (unsigned int n = 0; n < rest.size(); ++n) { char c = rest[rest.size() - n - 1]; unsigned char digit; - for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */ - if (base32Chars[digit] == c) break; + for (digit = 0; digit < nix32Chars.size(); ++digit) /* !!! slow */ + if (nix32Chars[digit] == c) break; if (digit >= 32) throw BadHash("invalid base-32 hash '%s'", rest); unsigned int b = n * 5; @@ -388,7 +389,11 @@ Hash compressHash(const Hash & hash, unsigned int newSize) std::optional parseHashFormatOpt(std::string_view hashFormatName) { if (hashFormatName == "base16") return HashFormat::Base16; - if (hashFormatName == "base32") return HashFormat::Base32; + if (hashFormatName == "nix32") return HashFormat::Nix32; + if (hashFormatName == "base32") { + warn(R"("base32" is a deprecated alias for hash format "nix32".)"); + return HashFormat::Nix32; + } if (hashFormatName == "base64") return HashFormat::Base64; if (hashFormatName == "sri") return HashFormat::SRI; return std::nullopt; @@ -407,8 +412,8 @@ std::string_view printHashFormat(HashFormat HashFormat) switch (HashFormat) { case HashFormat::Base64: return "base64"; - case HashFormat::Base32: - return "base32"; + case HashFormat::Nix32: + return "nix32"; case HashFormat::Base16: return "base16"; case HashFormat::SRI: diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 3c97ed4b1..7bed9e2bd 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -20,9 +20,9 @@ const int sha1HashSize = 20; const int sha256HashSize = 32; const int sha512HashSize = 64; -extern std::set hashAlgorithms; +extern const std::set hashAlgorithms; -extern const std::string base32Chars; +extern const std::string nix32Chars; /** * @brief Enumeration representing the hash formats. @@ -31,8 +31,8 @@ enum struct HashFormat : int { /// @brief Base 64 encoding. /// @see [IETF RFC 4648, section 4](https://datatracker.ietf.org/doc/html/rfc4648#section-4). Base64, - /// @brief Nix-specific base-32 encoding. @see base32Chars - Base32, + /// @brief Nix-specific base-32 encoding. @see nix32Chars + Nix32, /// @brief Lowercase hexadecimal encoding. @see base16Chars Base16, /// @brief ":", format of the SRI integrity attribute. @@ -40,6 +40,8 @@ enum struct HashFormat : int { SRI }; +extern const std::set hashFormats; + struct Hash { constexpr static size_t maxHashSize = 64; diff --git a/src/libutil/references.cc b/src/libutil/references.cc index d82d51945..b30e62c7b 100644 --- a/src/libutil/references.cc +++ b/src/libutil/references.cc @@ -23,8 +23,8 @@ static void search( static bool isBase32[256]; std::call_once(initialised, [](){ for (unsigned int i = 0; i < 256; ++i) isBase32[i] = false; - for (unsigned int i = 0; i < base32Chars.size(); ++i) - isBase32[(unsigned char) base32Chars[i]] = true; + for (unsigned int i = 0; i < nix32Chars.size(); ++i) + isBase32[(unsigned char) nix32Chars[i]] = true; }); for (size_t i = 0; i + refLength <= s.size(); ) { diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 75ad4e75f..db45be2a8 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -406,7 +406,7 @@ static void opQuery(Strings opFlags, Strings opArgs) auto info = store->queryPathInfo(j); if (query == qHash) { assert(info->narHash.algo == HashAlgorithm::SHA256); - cout << fmt("%s\n", info->narHash.to_string(HashFormat::Base32, true)); + cout << fmt("%s\n", info->narHash.to_string(HashFormat::Nix32, true)); } else if (query == qSize) cout << fmt("%d\n", info->narSize); } @@ -769,8 +769,8 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) if (current.first != info->narHash) { printError("path '%s' was modified! expected hash '%s', got '%s'", store->printStorePath(path), - info->narHash.to_string(HashFormat::Base32, true), - current.first.to_string(HashFormat::Base32, true)); + info->narHash.to_string(HashFormat::Nix32, true), + current.first.to_string(HashFormat::Nix32, true)); status = 1; } } @@ -898,7 +898,7 @@ static void opServe(Strings opFlags, Strings opArgs) out << info->narSize // downloadSize << info->narSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 4) - out << info->narHash.to_string(HashFormat::Base32, true) + out << info->narHash.to_string(HashFormat::Nix32, true) << renderContentAddress(info->ca) << info->sigs; } catch (InvalidPath &) { diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 173043c8a..f9c7592a3 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -39,7 +39,7 @@ struct CmdHashBase : Command addFlag({ .longName = "base32", .description = "Print the hash in base-32 (Nix-specific) format.", - .handler = {&hashFormat, HashFormat::Base32}, + .handler = {&hashFormat, HashFormat::Nix32}, }); addFlag({ @@ -120,7 +120,7 @@ struct CmdToBase : Command { return fmt("convert a hash to %s representation", hashFormat == HashFormat::Base16 ? "base-16" : - hashFormat == HashFormat::Base32 ? "base-32" : + hashFormat == HashFormat::Nix32 ? "base-32" : hashFormat == HashFormat::Base64 ? "base-64" : "SRI"); } @@ -143,24 +143,8 @@ struct CmdHashConvert : Command std::vector hashStrings; CmdHashConvert(): to(HashFormat::SRI) { - addFlag({ - .longName = "from", - // TODO: List format choices. Maybe introduce a constant? - .description = "The format of the input hash.", - .labels = {"hash format"}, - .handler = {[this](std::string str) { - from = parseHashFormat(str); - }}, - }); - addFlag({ - .longName = "to", - // TODO: List format choices. Maybe introduce a constant? - .description = "The format of the output hash.", - .labels = {"hash format"}, - .handler = {[this](std::string str) { - to = parseHashFormat(str); - }}, - }); + addFlag(Args::Flag::mkHashFormatOptFlag("from", &from)); + addFlag(Args::Flag::mkHashFormatFlagWithDefault("to", &to)); addFlag(Args::Flag::mkHashAlgoOptFlag("algo", &algo)); expectArgs({ .label = "hashes", @@ -170,7 +154,15 @@ struct CmdHashConvert : Command std::string description() override { - return "convert between different hash formats, e.g. base16, nix32, base64 and sri."; + std::string descr( "convert between different hash formats. Choose from: "); + auto iter = hashFormats.begin(); + assert(iter != hashFormats.end()); + descr += *iter++; + while (iter != hashFormats.end()) { + descr += ", " + *iter++; + } + + return descr; } Category category() override { return catUtility; } @@ -197,7 +189,7 @@ struct CmdHash : NixMultiCommand {"file", []() { return make_ref(FileIngestionMethod::Flat);; }}, {"path", []() { return make_ref(FileIngestionMethod::Recursive); }}, {"to-base16", []() { return make_ref(HashFormat::Base16); }}, - {"to-base32", []() { return make_ref(HashFormat::Base32); }}, + {"to-base32", []() { return make_ref(HashFormat::Nix32); }}, {"to-base64", []() { return make_ref(HashFormat::Base64); }}, {"to-sri", []() { return make_ref(HashFormat::SRI); }}, }) @@ -230,7 +222,7 @@ static int compatNixHash(int argc, char * * argv) printVersion("nix-hash"); else if (*arg == "--flat") flat = true; else if (*arg == "--base16") hashFormat = HashFormat::Base16; - else if (*arg == "--base32") hashFormat = HashFormat::Base32; + else if (*arg == "--base32") hashFormat = HashFormat::Nix32; else if (*arg == "--base64") hashFormat = HashFormat::Base64; else if (*arg == "--sri") hashFormat = HashFormat::SRI; else if (*arg == "--truncate") truncate = true; @@ -244,7 +236,7 @@ static int compatNixHash(int argc, char * * argv) } else if (*arg == "--to-base32") { op = opTo; - hashFormat = HashFormat::Base32; + hashFormat = HashFormat::Nix32; } else if (*arg == "--to-base64") { op = opTo; diff --git a/src/nix/verify.cc b/src/nix/verify.cc index cd0f6d95f..f0234f7be 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -109,8 +109,8 @@ struct CmdVerify : StorePathsCommand act2.result(resCorruptedPath, store->printStorePath(info->path)); printError("path '%s' was modified! expected hash '%s', got '%s'", store->printStorePath(info->path), - info->narHash.to_string(HashFormat::Base32, true), - hash.first.to_string(HashFormat::Base32, true)); + info->narHash.to_string(HashFormat::Nix32, true), + hash.first.to_string(HashFormat::Nix32, true)); } } diff --git a/tests/functional/hash.sh b/tests/functional/hash.sh index 031e33adf..278ed83b9 100644 --- a/tests/functional/hash.sh +++ b/tests/functional/hash.sh @@ -163,7 +163,7 @@ try3() { sri=$(nix hash convert --algo "$1" --from base16 "$2") [ "$sri" = "$1-$4" ] - sri=$(nix hash convert --algo "$1" --from base32 "$3") + sri=$(nix hash convert --algo "$1" --from nix32 "$3") [ "$sri" = "$1-$4" ] sri=$(nix hash convert --algo "$1" --from base64 "$4") [ "$sri" = "$1-$4" ] @@ -172,11 +172,11 @@ try3() { # Asserting input format fails. # - fail=$(nix hash convert --algo "$1" --from base32 "$2" 2>&1 || echo "exit: $?") + fail=$(nix hash convert --algo "$1" --from nix32 "$2" 2>&1 || echo "exit: $?") [[ "$fail" == "error: input hash"*"exit: 1" ]] fail=$(nix hash convert --algo "$1" --from base16 "$3" 2>&1 || echo "exit: $?") [[ "$fail" == "error: input hash"*"exit: 1" ]] - fail=$(nix hash convert --algo "$1" --from base32 "$4" 2>&1 || echo "exit: $?") + fail=$(nix hash convert --algo "$1" --from nix32 "$4" 2>&1 || echo "exit: $?") [[ "$fail" == "error: input hash"*"exit: 1" ]] } diff --git a/tests/functional/lang/eval-okay-convertHash.err.exp b/tests/functional/lang/eval-okay-convertHash.err.exp new file mode 100644 index 000000000..41d746725 --- /dev/null +++ b/tests/functional/lang/eval-okay-convertHash.err.exp @@ -0,0 +1,108 @@ +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". diff --git a/tests/functional/lang/eval-okay-convertHash.exp b/tests/functional/lang/eval-okay-convertHash.exp index 60e0a3c49..16b0240e5 100644 --- a/tests/functional/lang/eval-okay-convertHash.exp +++ b/tests/functional/lang/eval-okay-convertHash.exp @@ -1 +1 @@ -{ hashesBase16 = [ "d41d8cd98f00b204e9800998ecf8427e" "6c69ee7f211c640419d5366cc076ae46" "bb3438fbabd460ea6dbd27d153e2233b" "da39a3ee5e6b4b0d3255bfef95601890afd80709" "cd54e8568c1b37cf1e5badb0779bcbf382212189" "6d12e10b1d331dad210e47fd25d4f260802b7e77" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" "900a4469df00ccbfd0c145c6d1e4b7953dd0afafadd7534e3a4019e8d38fc663" "ad0387b3bd8652f730ca46d25f9c170af0fd589f42e7f23f5a9e6412d97d7e56" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" "9d0886f8c6b389398a16257bc79780fab9831c7fc11c8ab07fa732cb7b348feade382f92617c9c5305fefba0af02ab5fd39a587d330997ff5bd0db19f7666653" "21644b72aa259e5a588cd3afbafb1d4310f4889680f6c83b9d531596a5a284f34dbebff409d23bcc86aee6bad10c891606f075c6f4755cb536da27db5693f3a7" ]; hashesBase32 = [ "3y8bwfr609h3lh9ch0izcqq7fl" "26mrvc0v1nslch8r0w45zywsbc" "1v4gi57l97pmnylq6lmgxkhd5v" "143xibwh31h9bvxzalr0sjvbbvpa6ffs" "i4hj30pkrfdpgc5dbcgcydqviibfhm6d" "fxz2p030yba2bza71qhss79k3l5y24kd" "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73" "0qy6iz9yh6a079757mxdmypx0gcmnzjd3ij5q78bzk00vxll82lh" "0mkygpci4r4yb8zz5rs2kxcgvw0a2yf5zlj6r8qgfll6pnrqf0xd" "0zdl9zrg8r3i9c1g90lgg9ip5ijzv3yhz91i0zzn3r8ap9ws784gkp9dk9j3aglhgf1amqb0pj21mh7h1nxcl18akqvvf7ggqsy30yg" "19ncrpp37dx0nzzjw4k6zaqkb9mzaq2myhgpzh5aff7qqcj5wwdxslg6ixwncm7gyq8l761gwf87fgsh2bwfyr52s53k2dkqvw8c24x" "2kz74snvckxldmmbisz9ikmy031d28cs6xfdbl6rhxx42glpyz4vww4lajrc5akklxwixl0js4g84233pxvmbykiic5m7i5m9r4nr11" ]; hashesBase64 = [ "1B2M2Y8AsgTpgAmY7PhCfg==" "bGnufyEcZAQZ1TZswHauRg==" "uzQ4+6vUYOptvSfRU+IjOw==" "2jmj7l5rSw0yVb/vlWAYkK/YBwk=" "zVToVowbN88eW62wd5vL84IhIYk=" "bRLhCx0zHa0hDkf9JdTyYIArfnc=" "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" "kApEad8AzL/QwUXG0eS3lT3Qr6+t11NOOkAZ6NOPxmM=" "rQOHs72GUvcwykbSX5wXCvD9WJ9C5/I/Wp5kEtl9flY=" "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" "nQiG+MaziTmKFiV7x5eA+rmDHH/BHIqwf6cyy3s0j+reOC+SYXycUwX++6CvAqtf05pYfTMJl/9b0NsZ92ZmUw==" "IWRLcqolnlpYjNOvuvsdQxD0iJaA9sg7nVMVlqWihPNNvr/0CdI7zIau5rrRDIkWBvB1xvR1XLU22ifbVpPzpw==" ]; hashesSRI = [ "md5-1B2M2Y8AsgTpgAmY7PhCfg==" "md5-bGnufyEcZAQZ1TZswHauRg==" "md5-uzQ4+6vUYOptvSfRU+IjOw==" "sha1-2jmj7l5rSw0yVb/vlWAYkK/YBwk=" "sha1-zVToVowbN88eW62wd5vL84IhIYk=" "sha1-bRLhCx0zHa0hDkf9JdTyYIArfnc=" "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" "sha256-kApEad8AzL/QwUXG0eS3lT3Qr6+t11NOOkAZ6NOPxmM=" "sha256-rQOHs72GUvcwykbSX5wXCvD9WJ9C5/I/Wp5kEtl9flY=" "sha512-z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" "sha512-nQiG+MaziTmKFiV7x5eA+rmDHH/BHIqwf6cyy3s0j+reOC+SYXycUwX++6CvAqtf05pYfTMJl/9b0NsZ92ZmUw==" "sha512-IWRLcqolnlpYjNOvuvsdQxD0iJaA9sg7nVMVlqWihPNNvr/0CdI7zIau5rrRDIkWBvB1xvR1XLU22ifbVpPzpw==" ]; } +{ hashesBase16 = [ "d41d8cd98f00b204e9800998ecf8427e" "6c69ee7f211c640419d5366cc076ae46" "bb3438fbabd460ea6dbd27d153e2233b" "da39a3ee5e6b4b0d3255bfef95601890afd80709" "cd54e8568c1b37cf1e5badb0779bcbf382212189" "6d12e10b1d331dad210e47fd25d4f260802b7e77" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" "900a4469df00ccbfd0c145c6d1e4b7953dd0afafadd7534e3a4019e8d38fc663" "ad0387b3bd8652f730ca46d25f9c170af0fd589f42e7f23f5a9e6412d97d7e56" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" "9d0886f8c6b389398a16257bc79780fab9831c7fc11c8ab07fa732cb7b348feade382f92617c9c5305fefba0af02ab5fd39a587d330997ff5bd0db19f7666653" "21644b72aa259e5a588cd3afbafb1d4310f4889680f6c83b9d531596a5a284f34dbebff409d23bcc86aee6bad10c891606f075c6f4755cb536da27db5693f3a7" ]; hashesBase32 = [ "3y8bwfr609h3lh9ch0izcqq7fl" "26mrvc0v1nslch8r0w45zywsbc" "1v4gi57l97pmnylq6lmgxkhd5v" "143xibwh31h9bvxzalr0sjvbbvpa6ffs" "i4hj30pkrfdpgc5dbcgcydqviibfhm6d" "fxz2p030yba2bza71qhss79k3l5y24kd" "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73" "0qy6iz9yh6a079757mxdmypx0gcmnzjd3ij5q78bzk00vxll82lh" "0mkygpci4r4yb8zz5rs2kxcgvw0a2yf5zlj6r8qgfll6pnrqf0xd" "0zdl9zrg8r3i9c1g90lgg9ip5ijzv3yhz91i0zzn3r8ap9ws784gkp9dk9j3aglhgf1amqb0pj21mh7h1nxcl18akqvvf7ggqsy30yg" "19ncrpp37dx0nzzjw4k6zaqkb9mzaq2myhgpzh5aff7qqcj5wwdxslg6ixwncm7gyq8l761gwf87fgsh2bwfyr52s53k2dkqvw8c24x" "2kz74snvckxldmmbisz9ikmy031d28cs6xfdbl6rhxx42glpyz4vww4lajrc5akklxwixl0js4g84233pxvmbykiic5m7i5m9r4nr11" ]; hashesBase64 = [ "1B2M2Y8AsgTpgAmY7PhCfg==" "bGnufyEcZAQZ1TZswHauRg==" "uzQ4+6vUYOptvSfRU+IjOw==" "2jmj7l5rSw0yVb/vlWAYkK/YBwk=" "zVToVowbN88eW62wd5vL84IhIYk=" "bRLhCx0zHa0hDkf9JdTyYIArfnc=" "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" "kApEad8AzL/QwUXG0eS3lT3Qr6+t11NOOkAZ6NOPxmM=" "rQOHs72GUvcwykbSX5wXCvD9WJ9C5/I/Wp5kEtl9flY=" "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" "nQiG+MaziTmKFiV7x5eA+rmDHH/BHIqwf6cyy3s0j+reOC+SYXycUwX++6CvAqtf05pYfTMJl/9b0NsZ92ZmUw==" "IWRLcqolnlpYjNOvuvsdQxD0iJaA9sg7nVMVlqWihPNNvr/0CdI7zIau5rrRDIkWBvB1xvR1XLU22ifbVpPzpw==" ]; hashesNix32 = [ "3y8bwfr609h3lh9ch0izcqq7fl" "26mrvc0v1nslch8r0w45zywsbc" "1v4gi57l97pmnylq6lmgxkhd5v" "143xibwh31h9bvxzalr0sjvbbvpa6ffs" "i4hj30pkrfdpgc5dbcgcydqviibfhm6d" "fxz2p030yba2bza71qhss79k3l5y24kd" "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73" "0qy6iz9yh6a079757mxdmypx0gcmnzjd3ij5q78bzk00vxll82lh" "0mkygpci4r4yb8zz5rs2kxcgvw0a2yf5zlj6r8qgfll6pnrqf0xd" "0zdl9zrg8r3i9c1g90lgg9ip5ijzv3yhz91i0zzn3r8ap9ws784gkp9dk9j3aglhgf1amqb0pj21mh7h1nxcl18akqvvf7ggqsy30yg" "19ncrpp37dx0nzzjw4k6zaqkb9mzaq2myhgpzh5aff7qqcj5wwdxslg6ixwncm7gyq8l761gwf87fgsh2bwfyr52s53k2dkqvw8c24x" "2kz74snvckxldmmbisz9ikmy031d28cs6xfdbl6rhxx42glpyz4vww4lajrc5akklxwixl0js4g84233pxvmbykiic5m7i5m9r4nr11" ]; hashesSRI = [ "md5-1B2M2Y8AsgTpgAmY7PhCfg==" "md5-bGnufyEcZAQZ1TZswHauRg==" "md5-uzQ4+6vUYOptvSfRU+IjOw==" "sha1-2jmj7l5rSw0yVb/vlWAYkK/YBwk=" "sha1-zVToVowbN88eW62wd5vL84IhIYk=" "sha1-bRLhCx0zHa0hDkf9JdTyYIArfnc=" "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" "sha256-kApEad8AzL/QwUXG0eS3lT3Qr6+t11NOOkAZ6NOPxmM=" "sha256-rQOHs72GUvcwykbSX5wXCvD9WJ9C5/I/Wp5kEtl9flY=" "sha512-z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" "sha512-nQiG+MaziTmKFiV7x5eA+rmDHH/BHIqwf6cyy3s0j+reOC+SYXycUwX++6CvAqtf05pYfTMJl/9b0NsZ92ZmUw==" "sha512-IWRLcqolnlpYjNOvuvsdQxD0iJaA9sg7nVMVlqWihPNNvr/0CdI7zIau5rrRDIkWBvB1xvR1XLU22ifbVpPzpw==" ]; } diff --git a/tests/functional/lang/eval-okay-convertHash.nix b/tests/functional/lang/eval-okay-convertHash.nix index cf4909aaf..a0191ee8d 100644 --- a/tests/functional/lang/eval-okay-convertHash.nix +++ b/tests/functional/lang/eval-okay-convertHash.nix @@ -5,12 +5,14 @@ let map2' = f: fsts: snds: map2 f { inherit fsts snds; }; getOutputHashes = hashes: { hashesBase16 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base16";}) hashAlgos hashes; + hashesNix32 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "nix32";}) hashAlgos hashes; hashesBase32 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base32";}) hashAlgos hashes; hashesBase64 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base64";}) hashAlgos hashes; hashesSRI = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "sri" ;}) hashAlgos hashes; }; getOutputHashesColon = hashes: { hashesBase16 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "base16";}) hashAlgos hashes; + hashesNix32 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "nix32";}) hashAlgos hashes; hashesBase32 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "base32";}) hashAlgos hashes; hashesBase64 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "base64";}) hashAlgos hashes; hashesSRI = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "sri" ;}) hashAlgos hashes; diff --git a/tests/unit/libutil/hash.cc b/tests/unit/libutil/hash.cc index 4d82c7f09..a88994d0b 100644 --- a/tests/unit/libutil/hash.cc +++ b/tests/unit/libutil/hash.cc @@ -80,7 +80,7 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(hashFormat, testRoundTripPrintParse) { - for (const HashFormat hashFormat: { HashFormat::Base64, HashFormat::Base32, HashFormat::Base16, HashFormat::SRI}) { + for (const HashFormat hashFormat: { HashFormat::Base64, HashFormat::Nix32, HashFormat::Base16, HashFormat::SRI}) { ASSERT_EQ(parseHashFormat(printHashFormat(hashFormat)), hashFormat); ASSERT_EQ(*parseHashFormatOpt(printHashFormat(hashFormat)), hashFormat); } From 7ff876b92b590fd9559472935f4adce1d3d5efb7 Mon Sep 17 00:00:00 2001 From: Peter Kolloch Date: Tue, 28 Nov 2023 15:58:04 +0100 Subject: [PATCH 084/141] Add deprecation notice for old nix hash conversion subcommands. (But not yet nix-hash since `nix hash` is still hidden behind a feature flag.) https://github.com/NixOS/nix/issues/8876 --- src/nix/hash.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/nix/hash.cc b/src/nix/hash.cc index f9c7592a3..2c9deb0d5 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -127,6 +127,7 @@ struct CmdToBase : Command void run() override { + warn("The old format conversion sub commands of `nix hash` where deprecated in favor of `nix hash convert`."); for (auto s : args) logger->cout(Hash::parseAny(s, ht).to_string(hashFormat, hashFormat == HashFormat::SRI)); } @@ -208,6 +209,9 @@ static auto rCmdHash = registerCommand("hash"); /* Legacy nix-hash command. */ static int compatNixHash(int argc, char * * argv) { + // Wait until `nix hash convert` is not hidden behind experimental flags anymore. + // warn("`nix-hash` has been deprecated in favor of `nix hash convert`."); + std::optional ha; bool flat = false; HashFormat hashFormat = HashFormat::Base16; From 8afeaf05c4063d48e65d2d82c31c3323c3237f7c Mon Sep 17 00:00:00 2001 From: Peter Kolloch Date: Tue, 28 Nov 2023 19:02:15 +0100 Subject: [PATCH 085/141] Add docs/rl-notes for `nix hash convert` / `builtins.convertHash` https://github.com/NixOS/nix/issues/8876 --- doc/manual/rl-next/hash-format-nix32.md | 22 ++++++++++++ doc/manual/rl-next/nix-hash-convert.md | 47 +++++++++++++++++++++++++ src/libexpr/primops.cc | 8 ++--- src/nix/hash.cc | 2 +- 4 files changed, 74 insertions(+), 5 deletions(-) create mode 100644 doc/manual/rl-next/hash-format-nix32.md create mode 100644 doc/manual/rl-next/nix-hash-convert.md diff --git a/doc/manual/rl-next/hash-format-nix32.md b/doc/manual/rl-next/hash-format-nix32.md new file mode 100644 index 000000000..20c557da9 --- /dev/null +++ b/doc/manual/rl-next/hash-format-nix32.md @@ -0,0 +1,22 @@ +synopsis: Rename hash format `base32` to `nix32` +prs: #9452 +description: { + +Hash format `base32` was renamed to `nix32` since it used a special nix-specific character set for +[Base32](https://en.wikipedia.org/wiki/Base32). + +## Deprecation: Use `nix32` instead of `base32` as `toHashFormat` + +For the builtin `convertHash`, the `toHashFormat` parameter now accepts the same hash formats as the `--to`/`--from` +parameters of the `nix hash conert` command: `"base16"`, `"nix32"`, `"base64"`, and `"sri"`. The former `"base32"` value +remains as a deprecated alias for `"base32"`. Please convert your code from: + +```nix +builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base32";} +``` + +to + +```nix +builtins.convertHash { inherit hash hashAlgo; toHashFormat = "nix32";} +``` \ No newline at end of file diff --git a/doc/manual/rl-next/nix-hash-convert.md b/doc/manual/rl-next/nix-hash-convert.md new file mode 100644 index 000000000..de4367c5b --- /dev/null +++ b/doc/manual/rl-next/nix-hash-convert.md @@ -0,0 +1,47 @@ +synopsis: Add `nix hash convert` +prs: #9452 +description: { + +New [`nix hash convert`](https://github.com/NixOS/nix/issues/8876) sub command with a fast track +to stabilization! Examples: + +- Convert the hash to `nix32`. + + ```bash + $ nix hash convert --algo "sha1" --to nix32 "800d59cfcd3c05e900cb4e214be48f6b886a08df" + vw46m23bizj4n8afrc0fj19wrp7mj3c0 + ``` + `nix32` is a base32 encoding with a nix-specific character set. + Explicitly specify the hashing algorithm (optional with SRI hashes) but detect hash format by the length of the input + hash. +- Convert the hash to the `sri` format that includes an algorithm specification: + ```bash + nix hash convert --algo "sha1" "800d59cfcd3c05e900cb4e214be48f6b886a08df" + sha1-gA1Zz808BekAy04hS+SPa4hqCN8= + ``` + or with an explicit `-to` format: + ```bash + nix hash convert --algo "sha1" --to sri "800d59cfcd3c05e900cb4e214be48f6b886a08df" + sha1-gA1Zz808BekAy04hS+SPa4hqCN8= + ``` +- Assert the input format of the hash: + ```bash + nix hash convert --algo "sha256" --from nix32 "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=" + error: input hash 'ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=' does not have the expected format '--from nix32' + nix hash convert --algo "sha256" --from nix32 "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" + sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0= + ``` + +The `--to`/`--from`/`--algo` parameters have context-sensitive auto-completion. + +## Related Deprecations + +The following commands are still available but will emit a deprecation warning. Please convert your code to +`nix hash convert`: + +- `nix hash to-base16 $hash1 $hash2`: Use `nix hash convert --to base16 $hash1 $hash2` instead. +- `nix hash to-base32 $hash1 $hash2`: Use `nix hash convert --to nix32 $hash1 $hash2` instead. +- `nix hash to-base64 $hash1 $hash2`: Use `nix hash convert --to base64 $hash1 $hash2` instead. +- `nix hash to-sri $hash1 $hash2`: : Use `nix hash convert --to sri $hash1 $hash2` + or even just `nix hash convert $hash1 $hash2` instead. +} diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 7831f3803..4162a8da3 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1339,7 +1339,7 @@ drvName, Bindings * attrs, Value & v) .errPos = state.positions[noPos] }); - auto ht = parseHashAlgoOpt(outputHashAlgo).value_or(HashAlgorithm::SHA256); + auto ha = parseHashAlgoOpt(outputHashAlgo).value_or(HashAlgorithm::SHA256); auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive); for (auto & i : outputs) { @@ -1348,13 +1348,13 @@ drvName, Bindings * attrs, Value & v) drv.outputs.insert_or_assign(i, DerivationOutput::Impure { .method = method, - .hashAlgo = ht, + .hashAlgo = ha, }); else drv.outputs.insert_or_assign(i, DerivationOutput::CAFloating { .method = method, - .hashAlgo = ht, + .hashAlgo = ha, }); } } @@ -3837,7 +3837,7 @@ static RegisterPrimOp primop_convertHash({ The format of the resulting hash. Must be one of - `"base16"` - - `"base32"` + - `"nix32"` - `"base64"` - `"sri"` diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 2c9deb0d5..0bba3b7d2 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -118,7 +118,7 @@ struct CmdToBase : Command std::string description() override { - return fmt("convert a hash to %s representation", + return fmt("convert a hash to %s representation (deprecated, use `nix hash convert` instead)", hashFormat == HashFormat::Base16 ? "base-16" : hashFormat == HashFormat::Nix32 ? "base-32" : hashFormat == HashFormat::Base64 ? "base-64" : From d38ec1285573c98c987ec1421f7cec68754204f9 Mon Sep 17 00:00:00 2001 From: Peter Kolloch Date: Sat, 2 Dec 2023 11:53:50 +0100 Subject: [PATCH 086/141] Update src/libexpr/primops.cc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com> --- src/libexpr/primops.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 4162a8da3..828d118eb 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3838,6 +3838,7 @@ static RegisterPrimOp primop_convertHash({ The format of the resulting hash. Must be one of - `"base16"` - `"nix32"` + - `"base32"` (deprecated alias for `"nix32"`) - `"base64"` - `"sri"` From bbba2055f0b77e9677ef318ceea3084906eccd7d Mon Sep 17 00:00:00 2001 From: Peter Kolloch Date: Sat, 2 Dec 2023 16:43:52 +0100 Subject: [PATCH 087/141] Refactor concurrently added tests to use HashAlgorithm. https://github.com/NixOS/nix/issues/8876 --- tests/unit/libutil-support/tests/hash.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/libutil-support/tests/hash.cc b/tests/unit/libutil-support/tests/hash.cc index 577e9890e..50889cd33 100644 --- a/tests/unit/libutil-support/tests/hash.cc +++ b/tests/unit/libutil-support/tests/hash.cc @@ -11,7 +11,7 @@ using namespace nix; Gen Arbitrary::arbitrary() { - Hash hash(htSHA1); + Hash hash(HashAlgorithm::SHA1); for (size_t i = 0; i < hash.hashSize; ++i) hash.hash[i] = *gen::arbitrary(); return gen::just(hash); From e9a5365db66737d1438fd91eba6529d278e1efca Mon Sep 17 00:00:00 2001 From: Peter Kolloch Date: Sat, 2 Dec 2023 18:19:51 +0100 Subject: [PATCH 088/141] hash.sh: Make failure tests more tolerant of additional output "warning: you don'\''t have Internet access; disabling some network-dependent features" ... https://github.com/NixOS/nix/issues/8876 --- tests/functional/hash.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/functional/hash.sh b/tests/functional/hash.sh index 278ed83b9..47eed5178 100644 --- a/tests/functional/hash.sh +++ b/tests/functional/hash.sh @@ -173,11 +173,11 @@ try3() { # fail=$(nix hash convert --algo "$1" --from nix32 "$2" 2>&1 || echo "exit: $?") - [[ "$fail" == "error: input hash"*"exit: 1" ]] + [[ "$fail" == *"error: input hash"*"exit: 1" ]] fail=$(nix hash convert --algo "$1" --from base16 "$3" 2>&1 || echo "exit: $?") - [[ "$fail" == "error: input hash"*"exit: 1" ]] + [[ "$fail" == *"error: input hash"*"exit: 1" ]] fail=$(nix hash convert --algo "$1" --from nix32 "$4" 2>&1 || echo "exit: $?") - [[ "$fail" == "error: input hash"*"exit: 1" ]] + [[ "$fail" == *"error: input hash"*"exit: 1" ]] } From 9a1a3c43bf11912ad32c433219c4c21a1b6ca9dd Mon Sep 17 00:00:00 2001 From: Peter Kolloch Date: Sun, 3 Dec 2023 09:50:44 +0100 Subject: [PATCH 089/141] Store.xs: fix references to HashFormat::Nix32 https://github.com/NixOS/nix/issues/8876 --- perl/lib/Nix/Store.xs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 50148141b..82c7db608 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -78,7 +78,7 @@ SV * queryReferences(char * path) SV * queryPathHash(char * path) PPCODE: try { - auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(HashFormat::Base32, true); + auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(HashFormat::Nix32, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -104,7 +104,7 @@ SV * queryPathInfo(char * path, int base32) XPUSHs(&PL_sv_undef); else XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0))); - auto s = info->narHash.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, true); + auto s = info->narHash.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); @@ -206,7 +206,7 @@ SV * hashPath(char * algo, int base32, char * path) PPCODE: try { Hash h = hashPath(parseHashAlgo(algo), path).first; - auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false); + auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -217,7 +217,7 @@ SV * hashFile(char * algo, int base32, char * path) PPCODE: try { Hash h = hashFile(parseHashAlgo(algo), path); - auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false); + auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -228,7 +228,7 @@ SV * hashString(char * algo, int base32, char * s) PPCODE: try { Hash h = hashString(parseHashAlgo(algo), s); - auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false); + auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -239,7 +239,7 @@ SV * convertHash(char * algo, char * s, int toBase32) PPCODE: try { auto h = Hash::parseAny(s, parseHashAlgo(algo)); - auto s = h.to_string(toBase32 ? HashFormat::Base32 : HashFormat::Base16, false); + auto s = h.to_string(toBase32 ? HashFormat::Nix32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); From bf00d5ecef20c11eb7e49dff3482b9e536cf7abe Mon Sep 17 00:00:00 2001 From: Bryan Honof Date: Thu, 7 Dec 2023 11:04:48 +0100 Subject: [PATCH 090/141] fix(libutil/tarfile): add option to libarchive so it behaves correctly with AppleDouble files AppleDouble files were extracted differently on macOS machines than on other UNIX's. Setting `archive_read_set_format_option(this->archive, NULL ,"mac-ext",NULL)` fixes this problem, since it just ignores the AppleDouble file and treats it as a normal one. This was a problem since it caused source archives to be different between macOS and Linux. Ref: nixos/nix#9290 --- src/libutil/tarfile.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index 1733c791c..187b3e948 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -53,6 +53,7 @@ TarArchive::TarArchive(Source & source, bool raw) : buffer(65536) archive_read_support_format_raw(archive); archive_read_support_format_empty(archive); } + archive_read_set_option(archive, NULL, "mac-ext", NULL); check(archive_read_open(archive, (void *)this, callback_open, callback_read, callback_close), "Failed to open archive (%s)"); } @@ -63,6 +64,7 @@ TarArchive::TarArchive(const Path & path) archive_read_support_filter_all(archive); archive_read_support_format_all(archive); + archive_read_set_option(archive, NULL, "mac-ext", NULL); check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s"); } From a5521b7d9445af63a159d4fe7b44a0902c3a2a24 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 7 Dec 2023 10:49:29 -0500 Subject: [PATCH 091/141] Factor out `ServeProto::Serialiser` and test In the process, partially undo e89b5bd0bfeb4dfdd8fe7e6929544cb9ceb8a505 in that the ancient < 2.4 version is now supported again by the serializer again. `LegacySSHStore`, instead of also asserting that the version is at least 4, just checks that `narHash` is set. This allows us to better test the serializer in isolation for both versions (< 4 and >= 4). --- src/libstore/legacy-ssh-store.cc | 22 ++--- src/libstore/serve-protocol.cc | 44 ++++++++++ src/libstore/serve-protocol.hh | 3 + src/nix-store/nix-store.cc | 12 +-- .../unkeyed-valid-path-info-2.3.bin | Bin 0 -> 184 bytes .../unkeyed-valid-path-info-2.4.bin | Bin 0 -> 648 bytes tests/unit/libstore/serve-protocol.cc | 77 ++++++++++++++++++ 7 files changed, 131 insertions(+), 27 deletions(-) create mode 100644 tests/unit/libstore/data/serve-protocol/unkeyed-valid-path-info-2.3.bin create mode 100644 tests/unit/libstore/data/serve-protocol/unkeyed-valid-path-info-2.4.bin diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index fb1580dd6..277445ee6 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -172,24 +172,12 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor if (p.empty()) return callback(nullptr); auto path2 = parseStorePath(p); assert(path == path2); - /* Hash will be set below. FIXME construct ValidPathInfo at end. */ - auto info = std::make_shared(path, Hash::dummy); + auto info = std::make_shared( + path, + ServeProto::Serialise::read(*this, *conn)); - auto deriver = readString(conn->from); - if (deriver != "") - info->deriver = parseStorePath(deriver); - info->references = ServeProto::Serialise::read(*this, *conn); - readLongLong(conn->from); // download size - info->narSize = readLongLong(conn->from); - - { - auto s = readString(conn->from); - if (s == "") - throw Error("NAR hash is now mandatory"); - info->narHash = Hash::parseAnyPrefixed(s); - } - info->ca = ContentAddress::parseOpt(readString(conn->from)); - info->sigs = readStrings(conn->from); + if (info->narHash == Hash::dummy) + throw Error("NAR hash is now mandatory"); auto s = readString(conn->from); assert(s == ""); diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc index fb33553c5..c37b3095c 100644 --- a/src/libstore/serve-protocol.cc +++ b/src/libstore/serve-protocol.cc @@ -5,6 +5,7 @@ #include "serve-protocol.hh" #include "serve-protocol-impl.hh" #include "archive.hh" +#include "path-info.hh" #include @@ -54,4 +55,47 @@ void ServeProto::Serialise::write(const StoreDirConfig & store, Ser } } + +UnkeyedValidPathInfo ServeProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + /* Hash should be set below unless very old `nix-store --serve`. + Caller should assert that it did set it. */ + UnkeyedValidPathInfo info { Hash::dummy }; + + auto deriver = readString(conn.from); + if (deriver != "") + info.deriver = store.parseStorePath(deriver); + info.references = ServeProto::Serialise::read(store, conn); + + readLongLong(conn.from); // download size, unused + info.narSize = readLongLong(conn.from); + + if (GET_PROTOCOL_MINOR(conn.version) >= 4) { + auto s = readString(conn.from); + if (!s.empty()) + info.narHash = Hash::parseAnyPrefixed(s); + info.ca = ContentAddress::parseOpt(readString(conn.from)); + info.sigs = readStrings(conn.from); + } + + return info; +} + +void ServeProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const UnkeyedValidPathInfo & info) +{ + conn.to + << (info.deriver ? store.printStorePath(*info.deriver) : ""); + + ServeProto::write(store, conn, info.references); + // !!! Maybe we want compression? + conn.to + << info.narSize // downloadSize, lie a little + << info.narSize; + if (GET_PROTOCOL_MINOR(conn.version) >= 4) + conn.to + << info.narHash.to_string(HashFormat::Nix32, true) + << renderContentAddress(info.ca) + << info.sigs; +} + } diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index 6e9d66e2d..ada67a149 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -18,6 +18,7 @@ struct Source; // items being serialised struct BuildResult; +struct UnkeyedValidPathInfo; /** @@ -141,6 +142,8 @@ inline std::ostream & operator << (std::ostream & s, ServeProto::Command op) template<> DECLARE_SERVE_SERIALISER(BuildResult); +template<> +DECLARE_SERVE_SERIALISER(UnkeyedValidPathInfo); template DECLARE_SERVE_SERIALISER(std::vector); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index db45be2a8..45af7879c 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -891,16 +891,8 @@ static void opServe(Strings opFlags, Strings opArgs) for (auto & i : paths) { try { auto info = store->queryPathInfo(i); - out << store->printStorePath(info->path) - << (info->deriver ? store->printStorePath(*info->deriver) : ""); - ServeProto::write(*store, wconn, info->references); - // !!! Maybe we want compression? - out << info->narSize // downloadSize - << info->narSize; - if (GET_PROTOCOL_MINOR(clientVersion) >= 4) - out << info->narHash.to_string(HashFormat::Nix32, true) - << renderContentAddress(info->ca) - << info->sigs; + out << store->printStorePath(info->path); + ServeProto::write(*store, wconn, static_cast(*info)); } catch (InvalidPath &) { } } diff --git a/tests/unit/libstore/data/serve-protocol/unkeyed-valid-path-info-2.3.bin b/tests/unit/libstore/data/serve-protocol/unkeyed-valid-path-info-2.3.bin new file mode 100644 index 0000000000000000000000000000000000000000..8056ec055ed2039814ab2654aad984f07115518f GIT binary patch literal 184 zcmZQzKm~Rk5I&4HhDz(_Wmf1Hm*f|v>Zco)n`cxS7viFIlM;*cQi{sJIvJt*ahg+E XS&435ArJt~*Gcp=fmdD+P%du3+9=$K@X zTQa|lwn`cL3n8wg;O8!IE8WDiY-4aNzU{MP{{~7(rMCb8 literal 0 HcmV?d00001 diff --git a/tests/unit/libstore/serve-protocol.cc b/tests/unit/libstore/serve-protocol.cc index 6d2054f7d..c2298c6db 100644 --- a/tests/unit/libstore/serve-protocol.cc +++ b/tests/unit/libstore/serve-protocol.cc @@ -225,6 +225,83 @@ VERSIONED_CHARACTERIZATION_TEST( t; })) +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + unkeyedValidPathInfo_2_3, + "unkeyed-valid-path-info-2.3", + 2 << 8 | 3, + (std::tuple { + ({ + UnkeyedValidPathInfo info { Hash::dummy }; + info.narSize = 34878; + info; + }), + ({ + UnkeyedValidPathInfo info { Hash::dummy }; + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.references = { + StorePath { + "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo.drv", + }, + }; + info.narSize = 34878; + info; + }), + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + unkeyedValidPathInfo_2_4, + "unkeyed-valid-path-info-2.4", + 2 << 8 | 4, + (std::tuple { + ({ + UnkeyedValidPathInfo info { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.references = { + StorePath { + "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo.drv", + }, + }; + info.narSize = 34878; + info; + }), + ({ + ValidPathInfo info { + *LibStoreTest::store, + "foo", + FixedOutputInfo { + .method = FileIngestionMethod::Recursive, + .hash = hashString(HashAlgorithm::SHA256, "(...)"), + .references = { + .others = { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + }, + .self = true, + }, + }, + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.narSize = 34878; + info.sigs = { + "fake-sig-1", + "fake-sig-2", + }, + static_cast(std::move(info)); + }), + })) + VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, vector, From 0b80935c22f367b1deecffeddb97c90d7ed985e9 Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Thu, 7 Dec 2023 10:01:42 -0800 Subject: [PATCH 092/141] Pass positions when evaluating MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This includes position information in more places, making debugging easier. Before: ``` $ nix-instantiate --show-trace --eval tests/functional/lang/eval-fail-using-set-as-attr-name.nix error: … while evaluating an attribute name at «none»:0: (source not available) error: value is a set while a string was expected ``` After: ``` error: … while evaluating an attribute name at /pwd/lang/eval-fail-using-set-as-attr-name.nix:5:10: 4| in 5| attr.${key} | ^ 6| error: value is a set while a string was expected ``` --- .../rl-next/source-positions-in-errors.md | 45 +++++++++++++++++++ src/libexpr/eval-inline.hh | 12 ++--- src/libexpr/eval.cc | 18 ++++---- src/libexpr/nixexpr.hh | 1 + .../lang/eval-fail-attr-name-type.err.exp | 20 +++++++++ .../lang/eval-fail-attr-name-type.nix | 7 +++ .../lang/eval-fail-call-primop.err.exp | 12 +++++ .../functional/lang/eval-fail-call-primop.nix | 1 + .../lang/eval-fail-not-throws.err.exp | 18 ++++++++ .../functional/lang/eval-fail-not-throws.nix | 1 + .../eval-fail-using-set-as-attr-name.err.exp | 11 +++++ .../lang/eval-fail-using-set-as-attr-name.nix | 5 +++ 12 files changed, 137 insertions(+), 14 deletions(-) create mode 100644 doc/manual/rl-next/source-positions-in-errors.md create mode 100644 tests/functional/lang/eval-fail-attr-name-type.err.exp create mode 100644 tests/functional/lang/eval-fail-attr-name-type.nix create mode 100644 tests/functional/lang/eval-fail-call-primop.err.exp create mode 100644 tests/functional/lang/eval-fail-call-primop.nix create mode 100644 tests/functional/lang/eval-fail-not-throws.err.exp create mode 100644 tests/functional/lang/eval-fail-not-throws.nix create mode 100644 tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp create mode 100644 tests/functional/lang/eval-fail-using-set-as-attr-name.nix diff --git a/doc/manual/rl-next/source-positions-in-errors.md b/doc/manual/rl-next/source-positions-in-errors.md new file mode 100644 index 000000000..00f0b27e8 --- /dev/null +++ b/doc/manual/rl-next/source-positions-in-errors.md @@ -0,0 +1,45 @@ +synopsis: Source locations are printed more consistently in errors +issues: #561 +prs: #9555 +description: { + +Source location information is now included in error messages more +consistently. Given this code: + +```nix +let + attr = {foo = "bar";}; + key = {}; +in + attr.${key} +``` + +Previously, Nix would show this unhelpful message when attempting to evaluate +it: + +``` +error: + … while evaluating an attribute name + + at «none»:0: (source not available) + + error: value is a set while a string was expected +``` + +Now, the error message displays where the problematic value was found: + +``` +error: + … while evaluating an attribute name + + at bad.nix:4:11: + + 3| key = {}; + 4| in attr.${key} + | ^ + 5| + + error: value is a set while a string was expected +``` + +} diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index a988fa40c..c37b1d62b 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -103,8 +103,10 @@ void EvalState::forceValue(Value & v, Callable getPos) throw; } } - else if (v.isApp()) - callFunction(*v.app.left, *v.app.right, v, noPos); + else if (v.isApp()) { + PosIdx pos = getPos(); + callFunction(*v.app.left, *v.app.right, v, pos); + } else if (v.isBlackhole()) error("infinite recursion encountered").atPos(getPos()).template debugThrow(); } @@ -121,9 +123,9 @@ template [[gnu::always_inline]] inline void EvalState::forceAttrs(Value & v, Callable getPos, std::string_view errorCtx) { - forceValue(v, noPos); + PosIdx pos = getPos(); + forceValue(v, pos); if (v.type() != nAttrs) { - PosIdx pos = getPos(); error("value is %1% while a set was expected", showType(v)).withTrace(pos, errorCtx).debugThrow(); } } @@ -132,7 +134,7 @@ inline void EvalState::forceAttrs(Value & v, Callable getPos, std::string_view e [[gnu::always_inline]] inline void EvalState::forceList(Value & v, const PosIdx pos, std::string_view errorCtx) { - forceValue(v, noPos); + forceValue(v, pos); if (!v.isList()) { error("value is %1% while a list was expected", showType(v)).withTrace(pos, errorCtx).debugThrow(); } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7e68e6f9b..8a6e07fb0 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -344,7 +344,7 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env) } else { Value nameValue; name.expr->eval(state, env, nameValue); - state.forceStringNoCtx(nameValue, noPos, "while evaluating an attribute name"); + state.forceStringNoCtx(nameValue, name.expr->getPos(), "while evaluating an attribute name"); return state.symbols.create(nameValue.string_view()); } } @@ -1514,7 +1514,7 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v) e->eval(state, env, vTmp); for (auto & i : attrPath) { - state.forceValue(*vAttrs, noPos); + state.forceValue(*vAttrs, getPos()); Bindings::iterator j; auto name = getName(i, state, env); if (vAttrs->type() != nAttrs || @@ -1683,7 +1683,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & if (countCalls) primOpCalls[name]++; try { - vCur.primOp->fun(*this, noPos, args, vCur); + vCur.primOp->fun(*this, vCur.determinePos(noPos), args, vCur); } catch (Error & e) { addErrorTrace(e, pos, "while calling the '%1%' builtin", name); throw; @@ -1731,7 +1731,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & // 1. Unify this and above code. Heavily redundant. // 2. Create a fake env (arg1, arg2, etc.) and a fake expr (arg1: arg2: etc: builtins.name arg1 arg2 etc) // so the debugger allows to inspect the wrong parameters passed to the builtin. - primOp->primOp->fun(*this, noPos, vArgs, vCur); + primOp->primOp->fun(*this, vCur.determinePos(noPos), vArgs, vCur); } catch (Error & e) { addErrorTrace(e, pos, "while calling the '%1%' builtin", name); throw; @@ -1839,7 +1839,7 @@ https://nixos.org/manual/nix/stable/language/constructs.html#functions.)", symbo } } - callFunction(fun, allocValue()->mkAttrs(attrs), res, noPos); + callFunction(fun, allocValue()->mkAttrs(attrs), res, pos); } @@ -1875,7 +1875,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v) void ExprOpNot::eval(EvalState & state, Env & env, Value & v) { - v.mkBool(!state.evalBool(env, e, noPos, "in the argument of the not operator")); // XXX: FIXME: ! + v.mkBool(!state.evalBool(env, e, getPos(), "in the argument of the not operator")); // XXX: FIXME: ! } @@ -2316,7 +2316,7 @@ BackedStringView EvalState::coerceToString( std::string result; for (auto [n, v2] : enumerate(v.listItems())) { try { - result += *coerceToString(noPos, *v2, context, + result += *coerceToString(pos, *v2, context, "while evaluating one element of the list", coerceMore, copyToStore, canonicalizePath); } catch (Error & e) { @@ -2463,8 +2463,8 @@ SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value & bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx) { - forceValue(v1, noPos); - forceValue(v2, noPos); + forceValue(v1, pos); + forceValue(v2, pos); /* !!! Hack to support some old broken code that relies on pointer equality tests between sets. (Specifically, builderDefs calls diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 10099d49e..020286815 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -405,6 +405,7 @@ struct ExprOpNot : Expr { Expr * e; ExprOpNot(Expr * e) : e(e) { }; + PosIdx getPos() const override { return e->getPos(); } COMMON_METHODS }; diff --git a/tests/functional/lang/eval-fail-attr-name-type.err.exp b/tests/functional/lang/eval-fail-attr-name-type.err.exp new file mode 100644 index 000000000..5f9a073dd --- /dev/null +++ b/tests/functional/lang/eval-fail-attr-name-type.err.exp @@ -0,0 +1,20 @@ +error: + … while evaluating the attribute 'puppy."${key}"' + + at /pwd/lang/eval-fail-attr-name-type.nix:3:5: + + 2| attrs = { + 3| puppy.doggy = {}; + | ^ + 4| }; + + … while evaluating an attribute name + + at /pwd/lang/eval-fail-attr-name-type.nix:7:17: + + 6| in + 7| attrs.puppy.${key} + | ^ + 8| + + error: value is an integer while a string was expected diff --git a/tests/functional/lang/eval-fail-attr-name-type.nix b/tests/functional/lang/eval-fail-attr-name-type.nix new file mode 100644 index 000000000..a0e76004a --- /dev/null +++ b/tests/functional/lang/eval-fail-attr-name-type.nix @@ -0,0 +1,7 @@ +let + attrs = { + puppy.doggy = {}; + }; + key = 1; +in + attrs.puppy.${key} diff --git a/tests/functional/lang/eval-fail-call-primop.err.exp b/tests/functional/lang/eval-fail-call-primop.err.exp new file mode 100644 index 000000000..19b407c47 --- /dev/null +++ b/tests/functional/lang/eval-fail-call-primop.err.exp @@ -0,0 +1,12 @@ +error: + … while calling the 'length' builtin + + at /pwd/lang/eval-fail-call-primop.nix:1:1: + + 1| builtins.length 1 + | ^ + 2| + + … while evaluating the first argument passed to builtins.length + + error: value is an integer while a list was expected diff --git a/tests/functional/lang/eval-fail-call-primop.nix b/tests/functional/lang/eval-fail-call-primop.nix new file mode 100644 index 000000000..972eb72c7 --- /dev/null +++ b/tests/functional/lang/eval-fail-call-primop.nix @@ -0,0 +1 @@ +builtins.length 1 diff --git a/tests/functional/lang/eval-fail-not-throws.err.exp b/tests/functional/lang/eval-fail-not-throws.err.exp new file mode 100644 index 000000000..b290afb0a --- /dev/null +++ b/tests/functional/lang/eval-fail-not-throws.err.exp @@ -0,0 +1,18 @@ +error: + … in the argument of the not operator + + at /pwd/lang/eval-fail-not-throws.nix:1:4: + + 1| ! (throw "uh oh!") + | ^ + 2| + + … while calling the 'throw' builtin + + at /pwd/lang/eval-fail-not-throws.nix:1:4: + + 1| ! (throw "uh oh!") + | ^ + 2| + + error: uh oh! diff --git a/tests/functional/lang/eval-fail-not-throws.nix b/tests/functional/lang/eval-fail-not-throws.nix new file mode 100644 index 000000000..a74ce4ebe --- /dev/null +++ b/tests/functional/lang/eval-fail-not-throws.nix @@ -0,0 +1 @@ +! (throw "uh oh!") diff --git a/tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp b/tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp new file mode 100644 index 000000000..811d01b03 --- /dev/null +++ b/tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp @@ -0,0 +1,11 @@ +error: + … while evaluating an attribute name + + at /pwd/lang/eval-fail-using-set-as-attr-name.nix:5:10: + + 4| in + 5| attr.${key} + | ^ + 6| + + error: value is a set while a string was expected diff --git a/tests/functional/lang/eval-fail-using-set-as-attr-name.nix b/tests/functional/lang/eval-fail-using-set-as-attr-name.nix new file mode 100644 index 000000000..48e071a41 --- /dev/null +++ b/tests/functional/lang/eval-fail-using-set-as-attr-name.nix @@ -0,0 +1,5 @@ +let + attr = {foo = "bar";}; + key = {}; +in + attr.${key} From 96dd757b0c0f3d6702f8e38467a8bf467b43154e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 8 Dec 2023 00:44:55 -0500 Subject: [PATCH 093/141] Give `Derivation::tryResolve` an `evalStore` argument This is needed for building CA deriations with a src store / dest store split. In particular it is needed for Hydra. https://github.com/NixOS/hydra/issues/838 currently puts realizations, and thus build outputs, in the local store, but it should not. --- src/libstore/build/derivation-goal.cc | 2 +- src/libstore/derivations.cc | 4 ++-- src/libstore/derivations.hh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 81eef7c47..d4da374ba 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -558,7 +558,7 @@ void DerivationGoal::inputsRealised() inputDrvOutputs statefully, sometimes it gets out of sync with the real source of truth (store). So we query the store directly if there's a problem. */ - attempt = fullDrv.tryResolve(worker.store); + attempt = fullDrv.tryResolve(worker.store, &worker.evalStore); } assert(attempt); Derivation drvResolved { std::move(*attempt) }; diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 664ab7556..c35150b57 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -1002,13 +1002,13 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String } -std::optional Derivation::tryResolve(Store & store) const +std::optional Derivation::tryResolve(Store & store, Store * evalStore) const { std::map, StorePath> inputDrvOutputs; std::function::ChildNode &)> accum; accum = [&](auto & inputDrv, auto & node) { - for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(inputDrv)) { + for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(inputDrv, evalStore)) { if (outputPath) { inputDrvOutputs.insert_or_assign({inputDrv, outputName}, *outputPath); if (auto p = get(node.childMap, outputName)) diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 290abedcf..2a326b578 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -342,7 +342,7 @@ struct Derivation : BasicDerivation * 2. Input placeholders are replaced with realized input store * paths. */ - std::optional tryResolve(Store & store) const; + std::optional tryResolve(Store & store, Store * evalStore = nullptr) const; /** * Like the above, but instead of querying the Nix database for From f0ac2a35d5e9dfb3a53e6cc810e871fe119cbf4b Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Fri, 8 Dec 2023 11:36:57 -0500 Subject: [PATCH 094/141] Print the value in `error: cannot coerce` messages (#9553) * Print the value in `error: cannot coerce` messages This extends the `error: cannot coerce a TYPE to a string` message to print the value that could not be coerced. This helps with debugging by making it easier to track down where the value is being produced from, especially in errors with deep or unhelpful stack traces. Co-authored-by: Valentin Gagarin --- .../rl-next/print-value-in-coercion-error.md | 50 +++++++++++++++++++ .../src/language/string-interpolation.md | 2 +- src/libexpr/eval.cc | 10 ++-- ...al-fail-bad-string-interpolation-1.err.exp | 2 +- ...al-fail-bad-string-interpolation-3.err.exp | 2 +- tests/unit/libexpr/error_traces.cc | 28 +++++------ 6 files changed, 73 insertions(+), 21 deletions(-) create mode 100644 doc/manual/rl-next/print-value-in-coercion-error.md diff --git a/doc/manual/rl-next/print-value-in-coercion-error.md b/doc/manual/rl-next/print-value-in-coercion-error.md new file mode 100644 index 000000000..504ea67b9 --- /dev/null +++ b/doc/manual/rl-next/print-value-in-coercion-error.md @@ -0,0 +1,50 @@ +synopsis: Coercion errors include the failing value +issues: #561 +prs: #9553 +description: { + +The `error: cannot coerce a to a string` message now includes the value which caused the error. + +Previously, a failed string coercion produced a confusing error message if the trace didn't show where the offending value was defined: + +```bash +$ nix-instantiate --eval --expr ' +let x = { a = 1; }; in + +"${x}" +' +error: + … while evaluating a path segment + + at «string»:4:2: + + 3| + 4| "${x}" + | ^ + 5| + + error: cannot coerce a set to a string +``` + +Now, the error message includes the value itself: + +```bash +$ nix-instantiate --eval --expr ' +let x = { a = 1; }; in + +"${x}" +' +error: + … while evaluating a path segment + + at «string»:4:2: + + 3| + 4| "${x}" + | ^ + 5| + + error: cannot coerce a set to a string: { a = 1; } +``` + +} diff --git a/doc/manual/src/language/string-interpolation.md b/doc/manual/src/language/string-interpolation.md index e999b287b..6e28d2664 100644 --- a/doc/manual/src/language/string-interpolation.md +++ b/doc/manual/src/language/string-interpolation.md @@ -189,7 +189,7 @@ If neither is present, an error is thrown. > "${a}" > ``` > -> error: cannot coerce a set to a string +> error: cannot coerce a set to a string: { } > > at «string»:4:2: > diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7e68e6f9b..b52274b64 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -26,9 +26,9 @@ #include #include #include -#include #include #include +#include #include #include @@ -2286,7 +2286,7 @@ BackedStringView EvalState::coerceToString( return std::move(*maybeString); auto i = v.attrs->find(sOutPath); if (i == v.attrs->end()) { - error("cannot coerce %1% to a string", showType(v)) + error("cannot coerce %1% to a string: %2%", showType(v), printValue(*this, v)) .withTrace(pos, errorCtx) .debugThrow(); } @@ -2332,7 +2332,7 @@ BackedStringView EvalState::coerceToString( } } - error("cannot coerce %1% to a string", showType(v)) + error("cannot coerce %1% to a string: %2%", showType(v), printValue(*this, v)) .withTrace(pos, errorCtx) .debugThrow(); } @@ -2691,8 +2691,10 @@ void EvalState::printStatistics() std::string ExternalValueBase::coerceToString(const Pos & pos, NixStringContext & context, bool copyMore, bool copyToStore) const { + std::strstream printed; + print(printed); throw TypeError({ - .msg = hintfmt("cannot coerce %1% to a string", showType()) + .msg = hintfmt("cannot coerce %1% to a string: %2%", showType(), printed.str()) }); } diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp index eb73e9a52..e54ecc6d1 100644 --- a/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp @@ -7,4 +7,4 @@ error: | ^ 2| - error: cannot coerce a function to a string + error: cannot coerce a function to a string: diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp index ac14f329b..6f0a96f78 100644 --- a/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp @@ -7,4 +7,4 @@ error: | ^ 2| - error: cannot coerce a function to a string + error: cannot coerce a function to a string: diff --git a/tests/unit/libexpr/error_traces.cc b/tests/unit/libexpr/error_traces.cc index 81498f65a..c2403bee9 100644 --- a/tests/unit/libexpr/error_traces.cc +++ b/tests/unit/libexpr/error_traces.cc @@ -295,7 +295,7 @@ namespace nix { TEST_F(ErrorTraceTest, toPath) { ASSERT_TRACE2("toPath []", TypeError, - hintfmt("cannot coerce %s to a string", "a list"), + hintfmt("cannot coerce %s to a string: %s", "a list", "[ ]"), hintfmt("while evaluating the first argument passed to builtins.toPath")); ASSERT_TRACE2("toPath \"foo\"", @@ -309,7 +309,7 @@ namespace nix { TEST_F(ErrorTraceTest, storePath) { ASSERT_TRACE2("storePath true", TypeError, - hintfmt("cannot coerce %s to a string", "a Boolean"), + hintfmt("cannot coerce %s to a string: %s", "a Boolean", "true"), hintfmt("while evaluating the first argument passed to 'builtins.storePath'")); } @@ -318,7 +318,7 @@ namespace nix { TEST_F(ErrorTraceTest, pathExists) { ASSERT_TRACE2("pathExists []", TypeError, - hintfmt("cannot coerce %s to a string", "a list"), + hintfmt("cannot coerce %s to a string: %s", "a list", "[ ]"), hintfmt("while realising the context of a path")); ASSERT_TRACE2("pathExists \"zorglub\"", @@ -332,7 +332,7 @@ namespace nix { TEST_F(ErrorTraceTest, baseNameOf) { ASSERT_TRACE2("baseNameOf []", TypeError, - hintfmt("cannot coerce %s to a string", "a list"), + hintfmt("cannot coerce %s to a string: %s", "a list", "[ ]"), hintfmt("while evaluating the first argument passed to builtins.baseNameOf")); } @@ -377,7 +377,7 @@ namespace nix { TEST_F(ErrorTraceTest, filterSource) { ASSERT_TRACE2("filterSource [] []", TypeError, - hintfmt("cannot coerce %s to a string", "a list"), + hintfmt("cannot coerce %s to a string: %s", "a list", "[ ]"), hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); ASSERT_TRACE2("filterSource [] \"foo\"", @@ -1038,7 +1038,7 @@ namespace nix { TEST_F(ErrorTraceTest, toString) { ASSERT_TRACE2("toString { a = 1; }", TypeError, - hintfmt("cannot coerce %s to a string", "a set"), + hintfmt("cannot coerce %s to a string: %s", "a set", "{ a = 1; }"), hintfmt("while evaluating the first argument passed to builtins.toString")); } @@ -1057,7 +1057,7 @@ namespace nix { ASSERT_TRACE2("substring 0 3 {}", TypeError, - hintfmt("cannot coerce %s to a string", "a set"), + hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"), hintfmt("while evaluating the third argument (the string) passed to builtins.substring")); ASSERT_TRACE1("substring (-3) 3 \"sometext\"", @@ -1070,7 +1070,7 @@ namespace nix { TEST_F(ErrorTraceTest, stringLength) { ASSERT_TRACE2("stringLength {} # TODO: context is missing ???", TypeError, - hintfmt("cannot coerce %s to a string", "a set"), + hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"), hintfmt("while evaluating the argument passed to builtins.stringLength")); } @@ -1143,7 +1143,7 @@ namespace nix { ASSERT_TRACE2("concatStringsSep \"foo\" [ 1 2 {} ] # TODO: coerce to string is buggy", TypeError, - hintfmt("cannot coerce %s to a string", "an integer"), + hintfmt("cannot coerce %s to a string: %s", "an integer", "1"), hintfmt("while evaluating one element of the list of strings to concat passed to builtins.concatStringsSep")); } @@ -1229,12 +1229,12 @@ namespace nix { ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = {}; }", TypeError, - hintfmt("cannot coerce %s to a string", "a set"), + hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"), hintfmt("while evaluating the attribute 'system' of derivation 'foo'")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = {}; }", TypeError, - hintfmt("cannot coerce %s to a string", "a set"), + hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"), hintfmt("while evaluating the attribute 'outputs' of derivation 'foo'")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"drv\"; }", @@ -1279,17 +1279,17 @@ namespace nix { ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ {} ]; }", TypeError, - hintfmt("cannot coerce %s to a string", "a set"), + hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"), hintfmt("while evaluating an element of the argument list")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ \"a\" {} ]; }", TypeError, - hintfmt("cannot coerce %s to a string", "a set"), + hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"), hintfmt("while evaluating an element of the argument list")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; FOO = {}; }", TypeError, - hintfmt("cannot coerce %s to a string", "a set"), + hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"), hintfmt("while evaluating the attribute 'FOO' of derivation 'foo'")); } From f9ee1bedcf98334d8bc015c2e04e30fbba958a3e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 8 Dec 2023 13:18:52 -0500 Subject: [PATCH 095/141] Avoid `std::strstream`, fix the clang build According https://en.cppreference.com/w/cpp/io/strstream, it has been deprecated since C++98! The Clang + Linux build systems to not have it at all, or at least be hiding it. We can just use `std::stringstream` instead, I think. --- src/libexpr/eval.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index b52274b64..5d627224f 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -28,7 +28,7 @@ #include #include #include -#include +#include #include #include @@ -2691,7 +2691,7 @@ void EvalState::printStatistics() std::string ExternalValueBase::coerceToString(const Pos & pos, NixStringContext & context, bool copyMore, bool copyToStore) const { - std::strstream printed; + std::stringstream printed; print(printed); throw TypeError({ .msg = hintfmt("cannot coerce %1% to a string: %2%", showType(), printed.str()) From ce4ca574d24abe233b717babc679e4c9228ba94b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 6 Nov 2023 09:04:50 -0500 Subject: [PATCH 096/141] Clarify `SourceAccessor` methods should never implicitly follow symlinks The code has already been fixed (yay!) so what is left of this commit is just updating the API docs. Co-authored-by: Cole Helbling --- src/libutil/source-accessor.hh | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index 3ca12d624..4f4ff09c1 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -26,6 +26,13 @@ struct SourceAccessor /** * Return the contents of a file as a string. + * + * @note Unlike Unix, this method should *not* follow symlinks. Nix + * by default wants to manipulate symlinks explicitly, and not + * implictly follow them, as they are frequently untrusted user data + * and thus may point to arbitrary locations. Acting on the targets + * targets of symlinks should only occasionally be done, and only + * with care. */ virtual std::string readFile(const CanonPath & path); @@ -34,7 +41,10 @@ struct SourceAccessor * called with the size of the file before any data is written to * the sink. * - * Note: subclasses of `SourceAccessor` need to implement at least + * @note Like the other `readFile`, this method should *not* follow + * symlinks. + * + * @note subclasses of `SourceAccessor` need to implement at least * one of the `readFile()` variants. */ virtual void readFile( @@ -87,6 +97,9 @@ struct SourceAccessor typedef std::map DirEntries; + /** + * @note Like `readFile`, this method should *not* follow symlinks. + */ virtual DirEntries readDirectory(const CanonPath & path) = 0; virtual std::string readLink(const CanonPath & path) = 0; From 9b7b7a7561b24d48452627709e6872d9c610428b Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 9 Dec 2023 02:13:32 +0100 Subject: [PATCH 097/141] Revert "Print the value in `error: cannot coerce` messages (#9553)" This reverts commit f0ac2a35d5e9dfb3a53e6cc810e871fe119cbf4b. The request from the sibling PR, which also applies here, was not addressed. https://github.com/NixOS/nix/pull/9554#issuecomment-1845095735 --- .../rl-next/print-value-in-coercion-error.md | 50 ------------------- .../src/language/string-interpolation.md | 2 +- src/libexpr/eval.cc | 10 ++-- ...al-fail-bad-string-interpolation-1.err.exp | 2 +- ...al-fail-bad-string-interpolation-3.err.exp | 2 +- tests/unit/libexpr/error_traces.cc | 28 +++++------ 6 files changed, 21 insertions(+), 73 deletions(-) delete mode 100644 doc/manual/rl-next/print-value-in-coercion-error.md diff --git a/doc/manual/rl-next/print-value-in-coercion-error.md b/doc/manual/rl-next/print-value-in-coercion-error.md deleted file mode 100644 index 504ea67b9..000000000 --- a/doc/manual/rl-next/print-value-in-coercion-error.md +++ /dev/null @@ -1,50 +0,0 @@ -synopsis: Coercion errors include the failing value -issues: #561 -prs: #9553 -description: { - -The `error: cannot coerce a to a string` message now includes the value which caused the error. - -Previously, a failed string coercion produced a confusing error message if the trace didn't show where the offending value was defined: - -```bash -$ nix-instantiate --eval --expr ' -let x = { a = 1; }; in - -"${x}" -' -error: - … while evaluating a path segment - - at «string»:4:2: - - 3| - 4| "${x}" - | ^ - 5| - - error: cannot coerce a set to a string -``` - -Now, the error message includes the value itself: - -```bash -$ nix-instantiate --eval --expr ' -let x = { a = 1; }; in - -"${x}" -' -error: - … while evaluating a path segment - - at «string»:4:2: - - 3| - 4| "${x}" - | ^ - 5| - - error: cannot coerce a set to a string: { a = 1; } -``` - -} diff --git a/doc/manual/src/language/string-interpolation.md b/doc/manual/src/language/string-interpolation.md index 6e28d2664..e999b287b 100644 --- a/doc/manual/src/language/string-interpolation.md +++ b/doc/manual/src/language/string-interpolation.md @@ -189,7 +189,7 @@ If neither is present, an error is thrown. > "${a}" > ``` > -> error: cannot coerce a set to a string: { } +> error: cannot coerce a set to a string > > at «string»:4:2: > diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index c04e2d53d..841c223cd 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -27,9 +27,9 @@ #include #include #include +#include #include #include -#include #include #include @@ -2230,7 +2230,7 @@ BackedStringView EvalState::coerceToString( return std::move(*maybeString); auto i = v.attrs->find(sOutPath); if (i == v.attrs->end()) { - error("cannot coerce %1% to a string: %2%", showType(v), printValue(*this, v)) + error("cannot coerce %1% to a string", showType(v)) .withTrace(pos, errorCtx) .debugThrow(); } @@ -2276,7 +2276,7 @@ BackedStringView EvalState::coerceToString( } } - error("cannot coerce %1% to a string: %2%", showType(v), printValue(*this, v)) + error("cannot coerce %1% to a string", showType(v)) .withTrace(pos, errorCtx) .debugThrow(); } @@ -2635,10 +2635,8 @@ void EvalState::printStatistics() std::string ExternalValueBase::coerceToString(const Pos & pos, NixStringContext & context, bool copyMore, bool copyToStore) const { - std::stringstream printed; - print(printed); throw TypeError({ - .msg = hintfmt("cannot coerce %1% to a string: %2%", showType(), printed.str()) + .msg = hintfmt("cannot coerce %1% to a string", showType()) }); } diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp index e54ecc6d1..eb73e9a52 100644 --- a/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-1.err.exp @@ -7,4 +7,4 @@ error: | ^ 2| - error: cannot coerce a function to a string: + error: cannot coerce a function to a string diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp index 6f0a96f78..ac14f329b 100644 --- a/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-3.err.exp @@ -7,4 +7,4 @@ error: | ^ 2| - error: cannot coerce a function to a string: + error: cannot coerce a function to a string diff --git a/tests/unit/libexpr/error_traces.cc b/tests/unit/libexpr/error_traces.cc index c2403bee9..81498f65a 100644 --- a/tests/unit/libexpr/error_traces.cc +++ b/tests/unit/libexpr/error_traces.cc @@ -295,7 +295,7 @@ namespace nix { TEST_F(ErrorTraceTest, toPath) { ASSERT_TRACE2("toPath []", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a list", "[ ]"), + hintfmt("cannot coerce %s to a string", "a list"), hintfmt("while evaluating the first argument passed to builtins.toPath")); ASSERT_TRACE2("toPath \"foo\"", @@ -309,7 +309,7 @@ namespace nix { TEST_F(ErrorTraceTest, storePath) { ASSERT_TRACE2("storePath true", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a Boolean", "true"), + hintfmt("cannot coerce %s to a string", "a Boolean"), hintfmt("while evaluating the first argument passed to 'builtins.storePath'")); } @@ -318,7 +318,7 @@ namespace nix { TEST_F(ErrorTraceTest, pathExists) { ASSERT_TRACE2("pathExists []", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a list", "[ ]"), + hintfmt("cannot coerce %s to a string", "a list"), hintfmt("while realising the context of a path")); ASSERT_TRACE2("pathExists \"zorglub\"", @@ -332,7 +332,7 @@ namespace nix { TEST_F(ErrorTraceTest, baseNameOf) { ASSERT_TRACE2("baseNameOf []", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a list", "[ ]"), + hintfmt("cannot coerce %s to a string", "a list"), hintfmt("while evaluating the first argument passed to builtins.baseNameOf")); } @@ -377,7 +377,7 @@ namespace nix { TEST_F(ErrorTraceTest, filterSource) { ASSERT_TRACE2("filterSource [] []", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a list", "[ ]"), + hintfmt("cannot coerce %s to a string", "a list"), hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); ASSERT_TRACE2("filterSource [] \"foo\"", @@ -1038,7 +1038,7 @@ namespace nix { TEST_F(ErrorTraceTest, toString) { ASSERT_TRACE2("toString { a = 1; }", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a set", "{ a = 1; }"), + hintfmt("cannot coerce %s to a string", "a set"), hintfmt("while evaluating the first argument passed to builtins.toString")); } @@ -1057,7 +1057,7 @@ namespace nix { ASSERT_TRACE2("substring 0 3 {}", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"), + hintfmt("cannot coerce %s to a string", "a set"), hintfmt("while evaluating the third argument (the string) passed to builtins.substring")); ASSERT_TRACE1("substring (-3) 3 \"sometext\"", @@ -1070,7 +1070,7 @@ namespace nix { TEST_F(ErrorTraceTest, stringLength) { ASSERT_TRACE2("stringLength {} # TODO: context is missing ???", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"), + hintfmt("cannot coerce %s to a string", "a set"), hintfmt("while evaluating the argument passed to builtins.stringLength")); } @@ -1143,7 +1143,7 @@ namespace nix { ASSERT_TRACE2("concatStringsSep \"foo\" [ 1 2 {} ] # TODO: coerce to string is buggy", TypeError, - hintfmt("cannot coerce %s to a string: %s", "an integer", "1"), + hintfmt("cannot coerce %s to a string", "an integer"), hintfmt("while evaluating one element of the list of strings to concat passed to builtins.concatStringsSep")); } @@ -1229,12 +1229,12 @@ namespace nix { ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = {}; }", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"), + hintfmt("cannot coerce %s to a string", "a set"), hintfmt("while evaluating the attribute 'system' of derivation 'foo'")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = {}; }", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"), + hintfmt("cannot coerce %s to a string", "a set"), hintfmt("while evaluating the attribute 'outputs' of derivation 'foo'")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"drv\"; }", @@ -1279,17 +1279,17 @@ namespace nix { ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ {} ]; }", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"), + hintfmt("cannot coerce %s to a string", "a set"), hintfmt("while evaluating an element of the argument list")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ \"a\" {} ]; }", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"), + hintfmt("cannot coerce %s to a string", "a set"), hintfmt("while evaluating an element of the argument list")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; FOO = {}; }", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"), + hintfmt("cannot coerce %s to a string", "a set"), hintfmt("while evaluating the attribute 'FOO' of derivation 'foo'")); } From b9980b377ede0aca542b2baeeef9e4538dec20db Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 9 Dec 2023 02:36:33 +0100 Subject: [PATCH 098/141] Update rl-next/source-positions-in-errors for Nix 2.19+ --- doc/manual/rl-next/source-positions-in-errors.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/manual/rl-next/source-positions-in-errors.md b/doc/manual/rl-next/source-positions-in-errors.md index 00f0b27e8..15df884ea 100644 --- a/doc/manual/rl-next/source-positions-in-errors.md +++ b/doc/manual/rl-next/source-positions-in-errors.md @@ -21,8 +21,6 @@ it: error: … while evaluating an attribute name - at «none»:0: (source not available) - error: value is a set while a string was expected ``` From 6e8d5983143ae576e3f4b1d2954a5267f2943a49 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 9 Dec 2023 02:17:36 +0100 Subject: [PATCH 099/141] tests/lang/eval-fail-bad-string-interpolation-4: init --- .../lang/eval-fail-bad-string-interpolation-4.err.exp | 11 +++++++++++ .../lang/eval-fail-bad-string-interpolation-4.nix | 9 +++++++++ 2 files changed, 20 insertions(+) create mode 100644 tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp create mode 100644 tests/functional/lang/eval-fail-bad-string-interpolation-4.nix diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp new file mode 100644 index 000000000..07843a480 --- /dev/null +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp @@ -0,0 +1,11 @@ +error: + … while evaluating a path segment + + at /pwd/lang/eval-fail-bad-string-interpolation-4.nix:9:3: + + 8| # The error message should not be too long. + 9| ''${pkgs}'' + | ^ + 10| + + error: cannot coerce a set to a string diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-4.nix b/tests/functional/lang/eval-fail-bad-string-interpolation-4.nix new file mode 100644 index 000000000..457b5f06a --- /dev/null +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-4.nix @@ -0,0 +1,9 @@ +let + # Basically a "billion laughs" attack, but toned down to simulated `pkgs`. + ha = x: y: { a = x y; b = x y; c = x y; d = x y; e = x y; f = x y; g = x y; h = x y; j = x y; }; + has = ha (ha (ha (ha (x: x)))) "ha"; + # A large structure that has already been evaluated. + pkgs = builtins.deepSeq has has; +in +# The error message should not be too long. +''${pkgs}'' From 5417990e313272a5f1129ac39228b111e8dac857 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 8 Dec 2023 14:32:22 -0500 Subject: [PATCH 100/141] Create `ServeProto::BuildOptions` and a serializer for it More tests, and more serializers for Hydra reuse. --- src/libstore/legacy-ssh-store.cc | 22 +++----- src/libstore/serve-protocol.cc | 36 +++++++++++++ src/libstore/serve-protocol.hh | 25 +++++++++ src/nix-store/nix-store.cc | 34 ++++++++----- .../data/serve-protocol/build-options-2.1.bin | Bin 0 -> 16 bytes .../data/serve-protocol/build-options-2.2.bin | Bin 0 -> 24 bytes .../data/serve-protocol/build-options-2.3.bin | Bin 0 -> 40 bytes .../data/serve-protocol/build-options-2.7.bin | Bin 0 -> 48 bytes tests/unit/libstore/serve-protocol.cc | 48 ++++++++++++++++++ 9 files changed, 137 insertions(+), 28 deletions(-) create mode 100644 tests/unit/libstore/data/serve-protocol/build-options-2.1.bin create mode 100644 tests/unit/libstore/data/serve-protocol/build-options-2.2.bin create mode 100644 tests/unit/libstore/data/serve-protocol/build-options-2.3.bin create mode 100644 tests/unit/libstore/data/serve-protocol/build-options-2.7.bin diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 277445ee6..8ef2daa7b 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -275,20 +275,14 @@ private: void putBuildSettings(Connection & conn) { - conn.to - << settings.maxSilentTime - << settings.buildTimeout; - if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 2) - conn.to - << settings.maxLogSize; - if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3) - conn.to - << 0 // buildRepeat hasn't worked for ages anyway - << 0; - - if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 7) { - conn.to << ((int) settings.keepFailed); - } + ServeProto::write(*this, conn, ServeProto::BuildOptions { + .maxSilentTime = settings.maxSilentTime, + .buildTimeout = settings.buildTimeout, + .maxLogSize = settings.maxLogSize, + .nrRepeats = 0, // buildRepeat hasn't worked for ages anyway + .enforceDeterminism = 0, + .keepFailed = settings.keepFailed, + }); } public: diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc index c37b3095c..08bfad9e4 100644 --- a/src/libstore/serve-protocol.cc +++ b/src/libstore/serve-protocol.cc @@ -98,4 +98,40 @@ void ServeProto::Serialise::write(const StoreDirConfig & s << info.sigs; } + +ServeProto::BuildOptions ServeProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + BuildOptions options; + options.maxSilentTime = readInt(conn.from); + options.buildTimeout = readInt(conn.from); + if (GET_PROTOCOL_MINOR(conn.version) >= 2) + options.maxLogSize = readNum(conn.from); + if (GET_PROTOCOL_MINOR(conn.version) >= 3) { + options.nrRepeats = readInt(conn.from); + options.enforceDeterminism = readInt(conn.from); + } + if (GET_PROTOCOL_MINOR(conn.version) >= 7) { + options.keepFailed = (bool) readInt(conn.from); + } + return options; +} + +void ServeProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const ServeProto::BuildOptions & options) +{ + conn.to + << options.maxSilentTime + << options.buildTimeout; + if (GET_PROTOCOL_MINOR(conn.version) >= 2) + conn.to + << options.maxLogSize; + if (GET_PROTOCOL_MINOR(conn.version) >= 3) + conn.to + << options.nrRepeats + << options.enforceDeterminism; + + if (GET_PROTOCOL_MINOR(conn.version) >= 7) { + conn.to << ((int) options.keepFailed); + } +} + } diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index ada67a149..1665b935f 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -87,6 +87,13 @@ struct ServeProto { ServeProto::Serialise::write(store, conn, t); } + + /** + * Options for building shared between + * `ServeProto::Command::BuildPaths` and + * `ServeProto::Command::BuildDerivation`. + */ + struct BuildOptions; }; enum struct ServeProto::Command : uint64_t @@ -102,6 +109,22 @@ enum struct ServeProto::Command : uint64_t AddToStoreNar = 9, }; + +struct ServeProto::BuildOptions { + /** + * Default value in this and every other field is so tests pass when + * testing older deserialisers which do not set all the fields. + */ + time_t maxSilentTime = -1; + time_t buildTimeout = -1; + size_t maxLogSize = -1; + size_t nrRepeats = -1; + bool enforceDeterminism = -1; + bool keepFailed = -1; + + bool operator == (const ServeProto::BuildOptions &) const = default; +}; + /** * Convenience for sending operation codes. * @@ -144,6 +167,8 @@ template<> DECLARE_SERVE_SERIALISER(BuildResult); template<> DECLARE_SERVE_SERIALISER(UnkeyedValidPathInfo); +template<> +DECLARE_SERVE_SERIALISER(ServeProto::BuildOptions); template DECLARE_SERVE_SERIALISER(std::vector); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 45af7879c..d361dc0ac 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -835,27 +835,33 @@ static void opServe(Strings opFlags, Strings opArgs) verbosity = lvlError; settings.keepLog = false; settings.useSubstitutes = false; - settings.maxSilentTime = readInt(in); - settings.buildTimeout = readInt(in); + + auto options = ServeProto::Serialise::read(*store, rconn); + + // Only certain feilds get initialized based on the protocol + // version. This is why not all the code below is unconditional. + // See how the serialization logic in + // `ServeProto::Serialise` matches + // these conditions. + settings.maxSilentTime = options.maxSilentTime; + settings.buildTimeout = options.buildTimeout; if (GET_PROTOCOL_MINOR(clientVersion) >= 2) - settings.maxLogSize = readNum(in); + settings.maxLogSize = options.maxLogSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 3) { - auto nrRepeats = readInt(in); - if (nrRepeats != 0) { + if (options.nrRepeats != 0) { throw Error("client requested repeating builds, but this is not currently implemented"); } - // Ignore 'enforceDeterminism'. It used to be true by - // default, but also only never had any effect when - // `nrRepeats == 0`. We have already asserted that - // `nrRepeats` in fact is 0, so we can safely ignore this - // without doing something other than what the client - // asked for. - readInt(in); - + // Ignore 'options.enforceDeterminism'. + // + // It used to be true by default, but also only never had + // any effect when `nrRepeats == 0`. We have already + // checked that `nrRepeats` in fact is 0, so we can safely + // ignore this without doing something other than what the + // client asked for. settings.runDiffHook = true; } if (GET_PROTOCOL_MINOR(clientVersion) >= 7) { - settings.keepFailed = (bool) readInt(in); + settings.keepFailed = options.keepFailed; } }; diff --git a/tests/unit/libstore/data/serve-protocol/build-options-2.1.bin b/tests/unit/libstore/data/serve-protocol/build-options-2.1.bin new file mode 100644 index 0000000000000000000000000000000000000000..61e1d97286139e43918505b1b953128360d27853 GIT binary patch literal 16 NcmZQ&fB-fq4FCX;01N;C literal 0 HcmV?d00001 diff --git a/tests/unit/libstore/data/serve-protocol/build-options-2.2.bin b/tests/unit/libstore/data/serve-protocol/build-options-2.2.bin new file mode 100644 index 0000000000000000000000000000000000000000..045c2ff2b54ba708bc1d411f0e8786207c4e660a GIT binary patch literal 24 PcmZQ&fB-fq%?_mj0Vn_y literal 0 HcmV?d00001 diff --git a/tests/unit/libstore/data/serve-protocol/build-options-2.3.bin b/tests/unit/libstore/data/serve-protocol/build-options-2.3.bin new file mode 100644 index 0000000000000000000000000000000000000000..5c53458831dca70d5303363919f46f20f88993a2 GIT binary patch literal 40 VcmZQ&fB-fq%?_nGpfn?t1^@!!02}}S literal 0 HcmV?d00001 diff --git a/tests/unit/libstore/data/serve-protocol/build-options-2.7.bin b/tests/unit/libstore/data/serve-protocol/build-options-2.7.bin new file mode 100644 index 0000000000000000000000000000000000000000..1bc7b02db38f5f751c2610de84ff937e630567c9 GIT binary patch literal 48 WcmZQ&fB-fq%?_nGpfrqPgfajFxBwgg literal 0 HcmV?d00001 diff --git a/tests/unit/libstore/serve-protocol.cc b/tests/unit/libstore/serve-protocol.cc index c2298c6db..8f256d1e6 100644 --- a/tests/unit/libstore/serve-protocol.cc +++ b/tests/unit/libstore/serve-protocol.cc @@ -302,6 +302,54 @@ VERSIONED_CHARACTERIZATION_TEST( }), })) +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + build_options_2_1, + "build-options-2.1", + 2 << 8 | 1, + (ServeProto::BuildOptions { + .maxSilentTime = 5, + .buildTimeout = 6, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + build_options_2_2, + "build-options-2.2", + 2 << 8 | 2, + (ServeProto::BuildOptions { + .maxSilentTime = 5, + .buildTimeout = 6, + .maxLogSize = 7, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + build_options_2_3, + "build-options-2.3", + 2 << 8 | 3, + (ServeProto::BuildOptions { + .maxSilentTime = 5, + .buildTimeout = 6, + .maxLogSize = 7, + .nrRepeats = 8, + .enforceDeterminism = true, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + build_options_2_7, + "build-options-2.7", + 2 << 8 | 7, + (ServeProto::BuildOptions { + .maxSilentTime = 5, + .buildTimeout = 6, + .maxLogSize = 7, + .nrRepeats = 8, + .enforceDeterminism = false, + .keepFailed = true, + })) + VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, vector, From 360f3b3a9e0a74eb8b7d5a1744ad58f4cd487ca0 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 9 Dec 2023 19:50:33 +0100 Subject: [PATCH 101/141] changelog-d: Use roberth fork with markdown frontmatter support --- flake.nix | 2 +- misc/changelog-d.cabal.nix | 31 +++++++++++++++++++++++++++++++ misc/changelog-d.nix | 31 +++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 misc/changelog-d.cabal.nix create mode 100644 misc/changelog-d.nix diff --git a/flake.nix b/flake.nix index dbd45f053..90203e7d5 100644 --- a/flake.nix +++ b/flake.nix @@ -182,7 +182,7 @@ "--enable-internal-api-docs" ]; - changelog-d = pkgs.buildPackages.changelog-d; + changelog-d = pkgs.buildPackages.callPackage ./misc/changelog-d.nix { }; nativeBuildDeps = [ diff --git a/misc/changelog-d.cabal.nix b/misc/changelog-d.cabal.nix new file mode 100644 index 000000000..76f9353cd --- /dev/null +++ b/misc/changelog-d.cabal.nix @@ -0,0 +1,31 @@ +{ mkDerivation, aeson, base, bytestring, cabal-install-parsers +, Cabal-syntax, containers, directory, filepath, frontmatter +, generic-lens-lite, lib, mtl, optparse-applicative, parsec, pretty +, regex-applicative, text, pkgs +}: +let rev = "f30f6969e9cd8b56242309639d58acea21c99d06"; +in +mkDerivation { + pname = "changelog-d"; + version = "0.1"; + src = pkgs.fetchurl { + name = "changelog-d-${rev}.tar.gz"; + url = "https://codeberg.org/roberth/changelog-d/archive/${rev}.tar.gz"; + hash = "sha256-8a2+i5u7YoszAgd5OIEW0eYUcP8yfhtoOIhLJkylYJ4="; + } // { inherit rev; }; + isLibrary = false; + isExecutable = true; + libraryHaskellDepends = [ + aeson base bytestring cabal-install-parsers Cabal-syntax containers + directory filepath frontmatter generic-lens-lite mtl parsec pretty + regex-applicative text + ]; + executableHaskellDepends = [ + base bytestring Cabal-syntax directory filepath + optparse-applicative + ]; + doHaddock = false; + description = "Concatenate changelog entries into a single one"; + license = lib.licenses.gpl3Plus; + mainProgram = "changelog-d"; +} diff --git a/misc/changelog-d.nix b/misc/changelog-d.nix new file mode 100644 index 000000000..1b20f4596 --- /dev/null +++ b/misc/changelog-d.nix @@ -0,0 +1,31 @@ +# Taken temporarily from +{ + callPackage, + lib, + haskell, + haskellPackages, +}: + +let + hsPkg = haskellPackages.callPackage ./changelog-d.cabal.nix { }; + + addCompletions = haskellPackages.generateOptparseApplicativeCompletions ["changelog-d"]; + + haskellModifications = + lib.flip lib.pipe [ + addCompletions + haskell.lib.justStaticExecutables + ]; + + mkDerivationOverrides = finalAttrs: oldAttrs: { + + version = oldAttrs.version + "-git-${lib.strings.substring 0 7 oldAttrs.src.rev}"; + + meta = oldAttrs.meta // { + homepage = "https://codeberg.org/roberth/changelog-d"; + maintainers = [ lib.maintainers.roberth ]; + }; + + }; +in + (haskellModifications hsPkg).overrideAttrs mkDerivationOverrides From 3811b334c646bc3b4bf8caef6d13c9f5027246f1 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 9 Dec 2023 19:51:20 +0100 Subject: [PATCH 102/141] rl-next: Use markdown frontmatter syntax The old syntax is still supported, as long as you don't use a { in the description - the reason to migrate. --- doc/manual/rl-next/hash-format-nix32.md | 5 +++-- doc/manual/rl-next/mounted-ssh-store.md | 9 ++++----- doc/manual/rl-next/nix-config-show.md | 11 +++++------ doc/manual/rl-next/nix-env-json-drv-path.md | 9 +++------ doc/manual/rl-next/nix-hash-convert.md | 6 +++--- doc/manual/rl-next/source-positions-in-errors.md | 9 ++++----- doc/manual/src/contributing/hacking.md | 9 ++++----- 7 files changed, 26 insertions(+), 32 deletions(-) diff --git a/doc/manual/rl-next/hash-format-nix32.md b/doc/manual/rl-next/hash-format-nix32.md index 20c557da9..73e6fbb24 100644 --- a/doc/manual/rl-next/hash-format-nix32.md +++ b/doc/manual/rl-next/hash-format-nix32.md @@ -1,6 +1,7 @@ +--- synopsis: Rename hash format `base32` to `nix32` -prs: #9452 -description: { +prs: 9452 +--- Hash format `base32` was renamed to `nix32` since it used a special nix-specific character set for [Base32](https://en.wikipedia.org/wiki/Base32). diff --git a/doc/manual/rl-next/mounted-ssh-store.md b/doc/manual/rl-next/mounted-ssh-store.md index 39fac5283..6df44dbb6 100644 --- a/doc/manual/rl-next/mounted-ssh-store.md +++ b/doc/manual/rl-next/mounted-ssh-store.md @@ -1,9 +1,8 @@ +--- synopsis: Mounted SSH Store -issues: #7890 -prs: #7912 -description: { +issues: 7890 +prs: 7912 +--- Introduced the store [`mounted-ssh-ng://`](@docroot@/command-ref/new-cli/nix3-help-stores.md). This store allows full access to a Nix store on a remote machine and additionally requires that the store be mounted in the local filesystem. - -} diff --git a/doc/manual/rl-next/nix-config-show.md b/doc/manual/rl-next/nix-config-show.md index b2ad3c666..26b961b76 100644 --- a/doc/manual/rl-next/nix-config-show.md +++ b/doc/manual/rl-next/nix-config-show.md @@ -1,8 +1,7 @@ -synopsis: `nix config show` -issues: #7672 -prs: #9477 -description: { +--- +synopsis: Rename to `nix config show` +issues: 7672 +prs: 9477 +--- `nix show-config` was renamed to `nix config show`, and `nix doctor` was renamed to `nix config check`, to be more consistent with the rest of the command-line interface. - -} diff --git a/doc/manual/rl-next/nix-env-json-drv-path.md b/doc/manual/rl-next/nix-env-json-drv-path.md index fbe2b67d8..734cefd1b 100644 --- a/doc/manual/rl-next/nix-env-json-drv-path.md +++ b/doc/manual/rl-next/nix-env-json-drv-path.md @@ -1,9 +1,6 @@ +--- synopsis: Fix `nix-env --query --drv-path --json` -prs: #9257 -description: { +prs: 9257 +--- Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set. - -} - - diff --git a/doc/manual/rl-next/nix-hash-convert.md b/doc/manual/rl-next/nix-hash-convert.md index de4367c5b..2b718a66b 100644 --- a/doc/manual/rl-next/nix-hash-convert.md +++ b/doc/manual/rl-next/nix-hash-convert.md @@ -1,6 +1,7 @@ +--- synopsis: Add `nix hash convert` -prs: #9452 -description: { +prs: 9452 +--- New [`nix hash convert`](https://github.com/NixOS/nix/issues/8876) sub command with a fast track to stabilization! Examples: @@ -44,4 +45,3 @@ The following commands are still available but will emit a deprecation warning. - `nix hash to-base64 $hash1 $hash2`: Use `nix hash convert --to base64 $hash1 $hash2` instead. - `nix hash to-sri $hash1 $hash2`: : Use `nix hash convert --to sri $hash1 $hash2` or even just `nix hash convert $hash1 $hash2` instead. -} diff --git a/doc/manual/rl-next/source-positions-in-errors.md b/doc/manual/rl-next/source-positions-in-errors.md index 15df884ea..5b210289d 100644 --- a/doc/manual/rl-next/source-positions-in-errors.md +++ b/doc/manual/rl-next/source-positions-in-errors.md @@ -1,7 +1,8 @@ +--- synopsis: Source locations are printed more consistently in errors -issues: #561 -prs: #9555 -description: { +issues: 561 +prs: 9555 +--- Source location information is now included in error messages more consistently. Given this code: @@ -39,5 +40,3 @@ error: error: value is a set while a string was expected ``` - -} diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 9de5ad39b..237eff925 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -257,17 +257,16 @@ User-visible changes should come with a release note. Here's what a complete entry looks like. The file name is not incorporated in the document. ``` +--- synopsis: Basically a title -issues: #1234 -prs: #1238 -description: { +issues: 1234 +prs: 1238 +--- Here's one or more paragraphs that describe the change. - It's markdown - Add references to the manual using @docroot@ - -} ``` Significant changes should add the following header, which moves them to the top. From a856f603ed5a124f7eb818dadab6c88da73570fb Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 9 Dec 2023 19:55:47 +0100 Subject: [PATCH 103/141] Add checks.rl-next --- flake.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/flake.nix b/flake.nix index 90203e7d5..f499b0a9b 100644 --- a/flake.nix +++ b/flake.nix @@ -691,6 +691,11 @@ perlBindings = self.hydraJobs.perlBindings.${system}; installTests = self.hydraJobs.installTests.${system}; nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system}; + rl-next = + let pkgs = nixpkgsFor.${system}.native; + in pkgs.buildPackages.runCommand "test-rl-next-release-notes" { } '' + LANG=C.UTF-8 ${(commonDeps { inherit pkgs; }).changelog-d}/bin/changelog-d ${./doc/manual/rl-next} >$out + ''; } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) { dockerImage = self.hydraJobs.dockerImage.${system}; }); From a63be6578f7e17182fdec8e3d3fdbab19a814152 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 9 Dec 2023 21:22:20 +0100 Subject: [PATCH 104/141] flake.nix: Cache shell inputs through hydra --- flake.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flake.nix b/flake.nix index f499b0a9b..99480183a 100644 --- a/flake.nix +++ b/flake.nix @@ -540,6 +540,8 @@ # Binary package for various platforms. build = forAllSystems (system: self.packages.${system}.nix); + shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation); + buildStatic = lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static); buildCross = forAllCrossSystems (crossSystem: From 3c200da242d8f0ccda447866028bb757e0b0bbd9 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sun, 10 Dec 2023 06:16:32 +0100 Subject: [PATCH 105/141] document `fetchTree` (#9258) * document `fetchTree` * display experimental feature note at the top we have to enable the new `fetchTree` experimental feature to render it at all. this was a bug introduced when adding that new feature flag. Co-authored-by: tomberek Co-authored-by: Robert Hensing Co-authored-by: Silvan Mosberger --- doc/manual/generate-builtins.nix | 13 +- doc/manual/generate-settings.nix | 4 +- doc/manual/generate-store-info.nix | 4 +- src/libexpr/primops/fetchTree.cc | 242 ++++++++++++++++++++++++----- src/nix/main.cc | 1 + 5 files changed, 221 insertions(+), 43 deletions(-) diff --git a/doc/manual/generate-builtins.nix b/doc/manual/generate-builtins.nix index 05cae1c46..007b698f1 100644 --- a/doc/manual/generate-builtins.nix +++ b/doc/manual/generate-builtins.nix @@ -8,7 +8,15 @@ let showBuiltin = name: { doc, args, arity, experimental-feature }: let experimentalNotice = optionalString (experimental-feature != null) '' - This function is only available if the [${experimental-feature}](@docroot@/contributing/experimental-features.md#xp-feature-${experimental-feature}) experimental feature is enabled. + > **Note** + > + > This function is only available if the [`${experimental-feature}` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-${experimental-feature}) is enabled. + > + > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): + > + > ``` + > extra-experimental-features = ${experimental-feature} + > ``` ''; in squash '' @@ -17,10 +25,9 @@ let
- ${doc} - ${experimentalNotice} + ${doc}
''; listArgs = args: concatStringsSep " " (map (s: "${s}") args); diff --git a/doc/manual/generate-settings.nix b/doc/manual/generate-settings.nix index 74446b70b..504cda362 100644 --- a/doc/manual/generate-settings.nix +++ b/doc/manual/generate-settings.nix @@ -20,10 +20,10 @@ let else "`${setting}`"; # separate body to cleanly handle indentation body = '' - ${description} - ${experimentalFeatureNote} + ${description} + **Default:** ${showDefault documentDefault defaultValue} ${showAliases aliases} diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix index 57247a181..c311c3c39 100644 --- a/doc/manual/generate-store-info.nix +++ b/doc/manual/generate-store-info.nix @@ -19,10 +19,10 @@ let result = squash '' # ${name} - ${doc} - ${experimentalFeatureNote} + ${doc} + ## Settings ${showSettings { prefix = "store-${slug}"; inherit inlineHTML; } settings} diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 15f870a95..eb2df8626 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -187,45 +187,215 @@ static RegisterPrimOp primop_fetchTree({ .name = "fetchTree", .args = {"input"}, .doc = R"( - Fetch a source tree or a plain file using one of the supported backends. - *input* must be a [flake reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references), either in attribute set representation or in the URL-like syntax. - The input should be "locked", that is, it should contain a commit hash or content hash unless impure evaluation (`--impure`) is enabled. + Fetch a file system tree or a plain file using one of the supported backends and return an attribute set with: - > **Note** + - the resulting fixed-output [store path](@docroot@/glossary.md#gloss-store-path) + - the corresponding [NAR](@docroot@/glossary.md#gloss-nar) hash + - backend-specific metadata (currently not documented). + + *input* must be an attribute set with the following attributes: + + - `type` (String, required) + + One of the [supported source types](#source-types). + This determines other required and allowed input attributes. + + - `narHash` (String, optional) + + The `narHash` parameter can be used to substitute the source of the tree. + It also allows for verification of tree contents that may not be provided by the underlying transfer mechanism. + If `narHash` is set, the source is first looked up is the Nix store and [substituters](@docroot@/command-ref/conf-file.md#conf-substituters), and only fetched if not available. + + A subset of the output attributes of `fetchTree` can be re-used for subsequent calls to `fetchTree` to produce the same result again. + That is, `fetchTree` is idempotent. + + Downloads are cached in `$XDG_CACHE_HOME/nix`. + The remote source will be fetched from the network if both are true: + - A NAR hash is supplied and the corresponding store path is not [valid](@docroot@/glossary.md#gloss-validity), that is, not available in the store + + > **Note** + > + > [Substituters](@docroot@/command-ref/conf-file.md#conf-substituters) are not used in fetching. + + - There is no cache entry or the cache entry is older than [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl) + + ## Source types + + The following source types and associated input attributes are supported. + + + + - `"file"` + + Place a plain file into the Nix store. + This is similar to [`builtins.fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl) + + - `url` (String, required) + + Supported protocols: + + - `https` + + > **Example** + > + > ```nix + > fetchTree { + > type = "file"; + > url = "https://example.com/index.html"; + > } + > ``` + + - `http` + + Insecure HTTP transfer for legacy sources. + + > **Warning** + > + > HTTP performs no encryption or authentication. + > Use a `narHash` known in advance to ensure the output has expected contents. + + - `file` + + A file on the local file system. + + > **Example** + > + > ```nix + > fetchTree { + > type = "file"; + > url = "file:///home/eelco/nix/README.md"; + > } + > ``` + + - `"tarball"` + + Download a tar archive and extract it into the Nix store. + This has the same underyling implementation as [`builtins.fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball) + + - `url` (String, required) + + > **Example** + > + > ```nix + > fetchTree { + > type = "tarball"; + > url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11"; + > } + > ``` + + - `"git"` + + Fetch a Git tree and copy it to the Nix store. + This is similar to [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit). + + - `url` (String, required) + + The URL formats supported are the same as for Git itself. + + > **Example** + > + > ```nix + > fetchTree { + > type = "git"; + > url = "git@github.com:NixOS/nixpkgs.git"; + > } + > ``` + + > **Note** + > + > If the URL points to a local directory, and no `ref` or `rev` is given, Nix will only consider files added to the Git index, as listed by `git ls-files` but use the *current file contents* of the Git working directory. + + - `ref` (String, optional) + + A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name. + + Default: `"HEAD"` + + - `rev` (String, optional) + + A Git revision; a commit hash. + + Default: the tip of `ref` + + - `shallow` (Bool, optional) + + Make a shallow clone when fetching the Git tree. + + Default: `false` + + - `submodules` (Bool, optional) + + Also fetch submodules if available. + + Default: `false` + + - `allRefs` (Bool, optional) + + If set to `true`, always fetch the entire repository, even if the latest commit is still in the cache. + Otherwise, only the latest commit is fetched if it is not already cached. + + Default: `false` + + - `lastModified` (Integer, optional) + + Unix timestamp of the fetched commit. + + If set, pass through the value to the output attribute set. + Otherwise, generated from the fetched Git tree. + + - `revCount` (Integer, optional) + + Number of revisions in the history of the Git repository before the fetched commit. + + If set, pass through the value to the output attribute set. + Otherwise, generated from the fetched Git tree. + + The following input types are still subject to change: + + - `"path"` + - `"github"` + - `"gitlab"` + - `"sourcehut"` + - `"mercurial"` + + *input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references). + The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled. + + > **Example** > - > The URL-like syntax requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled. + > Fetch a GitHub repository using the attribute set representation: + > + > ```nix + > builtins.fetchTree { + > type = "github"; + > owner = "NixOS"; + > repo = "nixpkgs"; + > rev = "ae2e6b3958682513d28f7d633734571fb18285dd"; + > } + > ``` + > + > This evaluates to the following attribute set: + > + > ```nix + > { + > lastModified = 1686503798; + > lastModifiedDate = "20230611171638"; + > narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc="; + > outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source"; + > rev = "ae2e6b3958682513d28f7d633734571fb18285dd"; + > shortRev = "ae2e6b3"; + > } + > ``` - Here are some examples of how to use `fetchTree`: - - - Fetch a GitHub repository using the attribute set representation: - - ```nix - builtins.fetchTree { - type = "github"; - owner = "NixOS"; - repo = "nixpkgs"; - rev = "ae2e6b3958682513d28f7d633734571fb18285dd"; - } - ``` - - This evaluates to the following attribute set: - - ``` - { - lastModified = 1686503798; - lastModifiedDate = "20230611171638"; - narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc="; - outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source"; - rev = "ae2e6b3958682513d28f7d633734571fb18285dd"; - shortRev = "ae2e6b3"; - } - ``` - - - Fetch the same GitHub repository using the URL-like syntax: - - ``` - builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd" - ``` + > **Example** + > + > Fetch the same GitHub repository using the URL-like syntax: + > + > ```nix + > builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd" + > ``` )", .fun = prim_fetchTree, .experimentalFeature = Xp::FetchTree, diff --git a/src/nix/main.cc b/src/nix/main.cc index 109d2cc04..39c04069b 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -373,6 +373,7 @@ void mainWrapped(int argc, char * * argv) Xp::Flakes, Xp::FetchClosure, Xp::DynamicDerivations, + Xp::FetchTree, }; evalSettings.pureEval = false; EvalState state({}, openStore("dummy://")); From deadb3bfe9cde3e78e8e89340e4c92499069461a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 10 Dec 2023 14:28:14 -0500 Subject: [PATCH 106/141] Create header for `LegacySSHStore` In https://github.com/NixOS/nix/pull/6134#issuecomment-1079199888, @thuffschmitt proposed exposing `LegacySSHStore` in Nix for deduplication with Hydra, at least temporarily. I think that is a good idea. Note that the diff will look bad unless one ignores whitespace! Also try this locally: ```shell-session git diff --ignore-all-space HEAD^:src/libstore/legacy-ssh-store.cc HEAD:src/libstore/legacy-ssh-store.cc git diff --ignore-all-space HEAD^:src/libstore/legacy-ssh-store.cc HEAD:src/libstore/legacy-ssh-store.hh ``` --- src/libstore/legacy-ssh-store.cc | 726 ++++++++++++++----------------- src/libstore/legacy-ssh-store.hh | 132 ++++++ 2 files changed, 466 insertions(+), 392 deletions(-) create mode 100644 src/libstore/legacy-ssh-store.hh diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 8ef2daa7b..06bef9d08 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -1,3 +1,4 @@ +#include "legacy-ssh-store.hh" #include "ssh-store-config.hh" #include "archive.hh" #include "pool.hh" @@ -13,414 +14,355 @@ namespace nix { -struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig +std::string LegacySSHStoreConfig::doc() { - using CommonSSHStoreConfig::CommonSSHStoreConfig; + return + #include "legacy-ssh-store.md" + ; +} - const Setting remoteProgram{this, "nix-store", "remote-program", - "Path to the `nix-store` executable on the remote machine."}; - const Setting maxConnections{this, 1, "max-connections", - "Maximum number of concurrent SSH connections."}; - - const std::string name() override { return "SSH Store"; } - - std::string doc() override - { - return - #include "legacy-ssh-store.md" - ; - } -}; - -struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store +struct LegacySSHStore::Connection { - // Hack for getting remote build log output. - // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in - // the documentation - const Setting logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; - - struct Connection - { - std::unique_ptr sshConn; - FdSink to; - FdSource from; - ServeProto::Version remoteVersion; - bool good = true; - - /** - * Coercion to `ServeProto::ReadConn`. This makes it easy to use the - * factored out serve protocol searlizers with a - * `LegacySSHStore::Connection`. - * - * The serve protocol connection types are unidirectional, unlike - * this type. - */ - operator ServeProto::ReadConn () - { - return ServeProto::ReadConn { - .from = from, - .version = remoteVersion, - }; - } - - /* - * Coercion to `ServeProto::WriteConn`. This makes it easy to use the - * factored out serve protocol searlizers with a - * `LegacySSHStore::Connection`. - * - * The serve protocol connection types are unidirectional, unlike - * this type. - */ - operator ServeProto::WriteConn () - { - return ServeProto::WriteConn { - .to = to, - .version = remoteVersion, - }; - } - }; - - std::string host; - - ref> connections; - - SSHMaster master; - - static std::set uriSchemes() { return {"ssh"}; } - - LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params) - : StoreConfig(params) - , CommonSSHStoreConfig(params) - , LegacySSHStoreConfig(params) - , Store(params) - , host(host) - , connections(make_ref>( - std::max(1, (int) maxConnections), - [this]() { return openConnection(); }, - [](const ref & r) { return r->good; } - )) - , master( - host, - sshKey, - sshPublicHostKey, - // Use SSH master only if using more than 1 connection. - connections->capacity() > 1, - compress, - logFD) - { - } - - ref openConnection() - { - auto conn = make_ref(); - conn->sshConn = master.startCommand( - fmt("%s --serve --write", remoteProgram) - + (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get()))); - conn->to = FdSink(conn->sshConn->in.get()); - conn->from = FdSource(conn->sshConn->out.get()); - - try { - conn->to << SERVE_MAGIC_1 << SERVE_PROTOCOL_VERSION; - conn->to.flush(); - - StringSink saved; - try { - TeeSource tee(conn->from, saved); - unsigned int magic = readInt(tee); - if (magic != SERVE_MAGIC_2) - throw Error("'nix-store --serve' protocol mismatch from '%s'", host); - } catch (SerialisationError & e) { - /* In case the other side is waiting for our input, - close it. */ - conn->sshConn->in.close(); - auto msg = conn->from.drain(); - throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'", - host, chomp(saved.s + msg)); - } - conn->remoteVersion = readInt(conn->from); - if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200) - throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host); - - } catch (EndOfFile & e) { - throw Error("cannot connect to '%1%'", host); - } - - return conn; - }; - - std::string getUri() override - { - return *uriSchemes().begin() + "://" + host; - } - - void queryPathInfoUncached(const StorePath & path, - Callback> callback) noexcept override - { - try { - auto conn(connections->get()); - - /* No longer support missing NAR hash */ - assert(GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4); - - debug("querying remote host '%s' for info on '%s'", host, printStorePath(path)); - - conn->to << ServeProto::Command::QueryPathInfos << PathSet{printStorePath(path)}; - conn->to.flush(); - - auto p = readString(conn->from); - if (p.empty()) return callback(nullptr); - auto path2 = parseStorePath(p); - assert(path == path2); - auto info = std::make_shared( - path, - ServeProto::Serialise::read(*this, *conn)); - - if (info->narHash == Hash::dummy) - throw Error("NAR hash is now mandatory"); - - auto s = readString(conn->from); - assert(s == ""); - - callback(std::move(info)); - } catch (...) { callback.rethrow(); } - } - - void addToStore(const ValidPathInfo & info, Source & source, - RepairFlag repair, CheckSigsFlag checkSigs) override - { - debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host); - - auto conn(connections->get()); - - if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 5) { - - conn->to - << ServeProto::Command::AddToStoreNar - << printStorePath(info.path) - << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(HashFormat::Base16, false); - ServeProto::write(*this, *conn, info.references); - conn->to - << info.registrationTime - << info.narSize - << info.ultimate - << info.sigs - << renderContentAddress(info.ca); - try { - copyNAR(source, conn->to); - } catch (...) { - conn->good = false; - throw; - } - conn->to.flush(); - - } else { - - conn->to - << ServeProto::Command::ImportPaths - << 1; - try { - copyNAR(source, conn->to); - } catch (...) { - conn->good = false; - throw; - } - conn->to - << exportMagic - << printStorePath(info.path); - ServeProto::write(*this, *conn, info.references); - conn->to - << (info.deriver ? printStorePath(*info.deriver) : "") - << 0 - << 0; - conn->to.flush(); - - } - - if (readInt(conn->from) != 1) - throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host); - } - - void narFromPath(const StorePath & path, Sink & sink) override - { - auto conn(connections->get()); - - conn->to << ServeProto::Command::DumpStorePath << printStorePath(path); - conn->to.flush(); - copyNAR(conn->from, sink); - } - - std::optional queryPathFromHashPart(const std::string & hashPart) override - { unsupported("queryPathFromHashPart"); } - - StorePath addToStore( - std::string_view name, - const Path & srcPath, - FileIngestionMethod method, - HashAlgorithm hashAlgo, - PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) override - { unsupported("addToStore"); } - - StorePath addTextToStore( - std::string_view name, - std::string_view s, - const StorePathSet & references, - RepairFlag repair) override - { unsupported("addTextToStore"); } - -private: - - void putBuildSettings(Connection & conn) - { - ServeProto::write(*this, conn, ServeProto::BuildOptions { - .maxSilentTime = settings.maxSilentTime, - .buildTimeout = settings.buildTimeout, - .maxLogSize = settings.maxLogSize, - .nrRepeats = 0, // buildRepeat hasn't worked for ages anyway - .enforceDeterminism = 0, - .keepFailed = settings.keepFailed, - }); - } - -public: - - BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode) override - { - auto conn(connections->get()); - - conn->to - << ServeProto::Command::BuildDerivation - << printStorePath(drvPath); - writeDerivation(conn->to, *this, drv); - - putBuildSettings(*conn); - - conn->to.flush(); - - return ServeProto::Serialise::read(*this, *conn); - } - - void buildPaths(const std::vector & drvPaths, BuildMode buildMode, std::shared_ptr evalStore) override - { - if (evalStore && evalStore.get() != this) - throw Error("building on an SSH store is incompatible with '--eval-store'"); - - auto conn(connections->get()); - - conn->to << ServeProto::Command::BuildPaths; - Strings ss; - for (auto & p : drvPaths) { - auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p); - std::visit(overloaded { - [&](const StorePathWithOutputs & s) { - ss.push_back(s.to_string(*this)); - }, - [&](const StorePath & drvPath) { - throw Error("wanted to fetch '%s' but the legacy ssh protocol doesn't support merely substituting drv files via the build paths command. It would build them instead. Try using ssh-ng://", printStorePath(drvPath)); - }, - [&](std::monostate) { - throw Error("wanted build derivation that is itself a build product, but the legacy ssh protocol doesn't support that. Try using ssh-ng://"); - }, - }, sOrDrvPath); - } - conn->to << ss; - - putBuildSettings(*conn); - - conn->to.flush(); - - BuildResult result; - result.status = (BuildResult::Status) readInt(conn->from); - - if (!result.success()) { - conn->from >> result.errorMsg; - throw Error(result.status, result.errorMsg); - } - } - - void ensurePath(const StorePath & path) override - { unsupported("ensurePath"); } - - virtual ref getFSAccessor(bool requireValidPath) override - { unsupported("getFSAccessor"); } + std::unique_ptr sshConn; + FdSink to; + FdSource from; + ServeProto::Version remoteVersion; + bool good = true; /** - * The default instance would schedule the work on the client side, but - * for consistency with `buildPaths` and `buildDerivation` it should happen - * on the remote side. + * Coercion to `ServeProto::ReadConn`. This makes it easy to use the + * factored out serve protocol searlizers with a + * `LegacySSHStore::Connection`. * - * We make this fail for now so we can add implement this properly later - * without it being a breaking change. + * The serve protocol connection types are unidirectional, unlike + * this type. */ - void repairPath(const StorePath & path) override - { unsupported("repairPath"); } - - void computeFSClosure(const StorePathSet & paths, - StorePathSet & out, bool flipDirection = false, - bool includeOutputs = false, bool includeDerivers = false) override + operator ServeProto::ReadConn () { - if (flipDirection || includeDerivers) { - Store::computeFSClosure(paths, out, flipDirection, includeOutputs, includeDerivers); - return; - } - - auto conn(connections->get()); - - conn->to - << ServeProto::Command::QueryClosure - << includeOutputs; - ServeProto::write(*this, *conn, paths); - conn->to.flush(); - - for (auto & i : ServeProto::Serialise::read(*this, *conn)) - out.insert(i); + return ServeProto::ReadConn { + .from = from, + .version = remoteVersion, + }; } - StorePathSet queryValidPaths(const StorePathSet & paths, - SubstituteFlag maybeSubstitute = NoSubstitute) override - { - auto conn(connections->get()); - - conn->to - << ServeProto::Command::QueryValidPaths - << false // lock - << maybeSubstitute; - ServeProto::write(*this, *conn, paths); - conn->to.flush(); - - return ServeProto::Serialise::read(*this, *conn); - } - - void connect() override - { - auto conn(connections->get()); - } - - unsigned int getProtocol() override - { - auto conn(connections->get()); - return conn->remoteVersion; - } - - /** - * The legacy ssh protocol doesn't support checking for trusted-user. - * Try using ssh-ng:// instead if you want to know. + /* + * Coercion to `ServeProto::WriteConn`. This makes it easy to use the + * factored out serve protocol searlizers with a + * `LegacySSHStore::Connection`. + * + * The serve protocol connection types are unidirectional, unlike + * this type. */ - std::optional isTrustedClient() override + operator ServeProto::WriteConn () { - return std::nullopt; + return ServeProto::WriteConn { + .to = to, + .version = remoteVersion, + }; } - - void queryRealisationUncached(const DrvOutput &, - Callback> callback) noexcept override - // TODO: Implement - { unsupported("queryRealisation"); } }; + +LegacySSHStore::LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params) + : StoreConfig(params) + , CommonSSHStoreConfig(params) + , LegacySSHStoreConfig(params) + , Store(params) + , host(host) + , connections(make_ref>( + std::max(1, (int) maxConnections), + [this]() { return openConnection(); }, + [](const ref & r) { return r->good; } + )) + , master( + host, + sshKey, + sshPublicHostKey, + // Use SSH master only if using more than 1 connection. + connections->capacity() > 1, + compress, + logFD) +{ +} + + +ref LegacySSHStore::openConnection() +{ + auto conn = make_ref(); + conn->sshConn = master.startCommand( + fmt("%s --serve --write", remoteProgram) + + (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get()))); + conn->to = FdSink(conn->sshConn->in.get()); + conn->from = FdSource(conn->sshConn->out.get()); + + try { + conn->to << SERVE_MAGIC_1 << SERVE_PROTOCOL_VERSION; + conn->to.flush(); + + StringSink saved; + try { + TeeSource tee(conn->from, saved); + unsigned int magic = readInt(tee); + if (magic != SERVE_MAGIC_2) + throw Error("'nix-store --serve' protocol mismatch from '%s'", host); + } catch (SerialisationError & e) { + /* In case the other side is waiting for our input, + close it. */ + conn->sshConn->in.close(); + auto msg = conn->from.drain(); + throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'", + host, chomp(saved.s + msg)); + } + conn->remoteVersion = readInt(conn->from); + if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200) + throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host); + + } catch (EndOfFile & e) { + throw Error("cannot connect to '%1%'", host); + } + + return conn; +}; + + +std::string LegacySSHStore::getUri() +{ + return *uriSchemes().begin() + "://" + host; +} + + +void LegacySSHStore::queryPathInfoUncached(const StorePath & path, + Callback> callback) noexcept +{ + try { + auto conn(connections->get()); + + /* No longer support missing NAR hash */ + assert(GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4); + + debug("querying remote host '%s' for info on '%s'", host, printStorePath(path)); + + conn->to << ServeProto::Command::QueryPathInfos << PathSet{printStorePath(path)}; + conn->to.flush(); + + auto p = readString(conn->from); + if (p.empty()) return callback(nullptr); + auto path2 = parseStorePath(p); + assert(path == path2); + auto info = std::make_shared( + path, + ServeProto::Serialise::read(*this, *conn)); + + if (info->narHash == Hash::dummy) + throw Error("NAR hash is now mandatory"); + + auto s = readString(conn->from); + assert(s == ""); + + callback(std::move(info)); + } catch (...) { callback.rethrow(); } +} + + +void LegacySSHStore::addToStore(const ValidPathInfo & info, Source & source, + RepairFlag repair, CheckSigsFlag checkSigs) +{ + debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host); + + auto conn(connections->get()); + + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 5) { + + conn->to + << ServeProto::Command::AddToStoreNar + << printStorePath(info.path) + << (info.deriver ? printStorePath(*info.deriver) : "") + << info.narHash.to_string(HashFormat::Base16, false); + ServeProto::write(*this, *conn, info.references); + conn->to + << info.registrationTime + << info.narSize + << info.ultimate + << info.sigs + << renderContentAddress(info.ca); + try { + copyNAR(source, conn->to); + } catch (...) { + conn->good = false; + throw; + } + conn->to.flush(); + + } else { + + conn->to + << ServeProto::Command::ImportPaths + << 1; + try { + copyNAR(source, conn->to); + } catch (...) { + conn->good = false; + throw; + } + conn->to + << exportMagic + << printStorePath(info.path); + ServeProto::write(*this, *conn, info.references); + conn->to + << (info.deriver ? printStorePath(*info.deriver) : "") + << 0 + << 0; + conn->to.flush(); + + } + + if (readInt(conn->from) != 1) + throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host); +} + + +void LegacySSHStore::narFromPath(const StorePath & path, Sink & sink) +{ + auto conn(connections->get()); + + conn->to << ServeProto::Command::DumpStorePath << printStorePath(path); + conn->to.flush(); + copyNAR(conn->from, sink); +} + + +void LegacySSHStore::putBuildSettings(Connection & conn) +{ + ServeProto::write(*this, conn, ServeProto::BuildOptions { + .maxSilentTime = settings.maxSilentTime, + .buildTimeout = settings.buildTimeout, + .maxLogSize = settings.maxLogSize, + .nrRepeats = 0, // buildRepeat hasn't worked for ages anyway + .enforceDeterminism = 0, + .keepFailed = settings.keepFailed, + }); +} + + +BuildResult LegacySSHStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, + BuildMode buildMode) +{ + auto conn(connections->get()); + + conn->to + << ServeProto::Command::BuildDerivation + << printStorePath(drvPath); + writeDerivation(conn->to, *this, drv); + + putBuildSettings(*conn); + + conn->to.flush(); + + return ServeProto::Serialise::read(*this, *conn); +} + + +void LegacySSHStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode, std::shared_ptr evalStore) +{ + if (evalStore && evalStore.get() != this) + throw Error("building on an SSH store is incompatible with '--eval-store'"); + + auto conn(connections->get()); + + conn->to << ServeProto::Command::BuildPaths; + Strings ss; + for (auto & p : drvPaths) { + auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p); + std::visit(overloaded { + [&](const StorePathWithOutputs & s) { + ss.push_back(s.to_string(*this)); + }, + [&](const StorePath & drvPath) { + throw Error("wanted to fetch '%s' but the legacy ssh protocol doesn't support merely substituting drv files via the build paths command. It would build them instead. Try using ssh-ng://", printStorePath(drvPath)); + }, + [&](std::monostate) { + throw Error("wanted build derivation that is itself a build product, but the legacy ssh protocol doesn't support that. Try using ssh-ng://"); + }, + }, sOrDrvPath); + } + conn->to << ss; + + putBuildSettings(*conn); + + conn->to.flush(); + + BuildResult result; + result.status = (BuildResult::Status) readInt(conn->from); + + if (!result.success()) { + conn->from >> result.errorMsg; + throw Error(result.status, result.errorMsg); + } +} + + +void LegacySSHStore::computeFSClosure(const StorePathSet & paths, + StorePathSet & out, bool flipDirection, + bool includeOutputs, bool includeDerivers) +{ + if (flipDirection || includeDerivers) { + Store::computeFSClosure(paths, out, flipDirection, includeOutputs, includeDerivers); + return; + } + + auto conn(connections->get()); + + conn->to + << ServeProto::Command::QueryClosure + << includeOutputs; + ServeProto::write(*this, *conn, paths); + conn->to.flush(); + + for (auto & i : ServeProto::Serialise::read(*this, *conn)) + out.insert(i); +} + + +StorePathSet LegacySSHStore::queryValidPaths(const StorePathSet & paths, + SubstituteFlag maybeSubstitute) +{ + auto conn(connections->get()); + + conn->to + << ServeProto::Command::QueryValidPaths + << false // lock + << maybeSubstitute; + ServeProto::write(*this, *conn, paths); + conn->to.flush(); + + return ServeProto::Serialise::read(*this, *conn); +} + + +void LegacySSHStore::connect() +{ + auto conn(connections->get()); +} + + +unsigned int LegacySSHStore::getProtocol() +{ + auto conn(connections->get()); + return conn->remoteVersion; +} + + +/** + * The legacy ssh protocol doesn't support checking for trusted-user. + * Try using ssh-ng:// instead if you want to know. + */ +std::optional isTrustedClient() +{ + return std::nullopt; +} + + static RegisterStoreImplementation regLegacySSHStore; } diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh new file mode 100644 index 000000000..c40c256bb --- /dev/null +++ b/src/libstore/legacy-ssh-store.hh @@ -0,0 +1,132 @@ +#pragma once +///@file + +#include "ssh-store-config.hh" +#include "store-api.hh" +#include "ssh.hh" +#include "callback.hh" +#include "pool.hh" + +namespace nix { + +struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig +{ + using CommonSSHStoreConfig::CommonSSHStoreConfig; + + const Setting remoteProgram{this, "nix-store", "remote-program", + "Path to the `nix-store` executable on the remote machine."}; + + const Setting maxConnections{this, 1, "max-connections", + "Maximum number of concurrent SSH connections."}; + + const std::string name() override { return "SSH Store"; } + + std::string doc() override; +}; + +struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store +{ + // Hack for getting remote build log output. + // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in + // the documentation + const Setting logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; + + struct Connection; + + std::string host; + + ref> connections; + + SSHMaster master; + + static std::set uriSchemes() { return {"ssh"}; } + + LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params); + + ref openConnection(); + + std::string getUri() override; + + void queryPathInfoUncached(const StorePath & path, + Callback> callback) noexcept override; + + void addToStore(const ValidPathInfo & info, Source & source, + RepairFlag repair, CheckSigsFlag checkSigs) override; + + void narFromPath(const StorePath & path, Sink & sink) override; + + std::optional queryPathFromHashPart(const std::string & hashPart) override + { unsupported("queryPathFromHashPart"); } + + StorePath addToStore( + std::string_view name, + const Path & srcPath, + FileIngestionMethod method, + HashAlgorithm hashAlgo, + PathFilter & filter, + RepairFlag repair, + const StorePathSet & references) override + { unsupported("addToStore"); } + + StorePath addTextToStore( + std::string_view name, + std::string_view s, + const StorePathSet & references, + RepairFlag repair) override + { unsupported("addTextToStore"); } + +private: + + void putBuildSettings(Connection & conn); + +public: + + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, + BuildMode buildMode) override; + + void buildPaths(const std::vector & drvPaths, BuildMode buildMode, std::shared_ptr evalStore) override; + + void ensurePath(const StorePath & path) override + { unsupported("ensurePath"); } + + virtual ref getFSAccessor(bool requireValidPath) override + { unsupported("getFSAccessor"); } + + /** + * The default instance would schedule the work on the client side, but + * for consistency with `buildPaths` and `buildDerivation` it should happen + * on the remote side. + * + * We make this fail for now so we can add implement this properly later + * without it being a breaking change. + */ + void repairPath(const StorePath & path) override + { unsupported("repairPath"); } + + void computeFSClosure(const StorePathSet & paths, + StorePathSet & out, bool flipDirection = false, + bool includeOutputs = false, bool includeDerivers = false) override; + + StorePathSet queryValidPaths(const StorePathSet & paths, + SubstituteFlag maybeSubstitute = NoSubstitute) override; + + void connect() override; + + unsigned int getProtocol() override; + + /** + * The legacy ssh protocol doesn't support checking for trusted-user. + * Try using ssh-ng:// instead if you want to know. + */ + std::optional isTrustedClient() override + { + return std::nullopt; + } + + void queryRealisationUncached(const DrvOutput &, + Callback> callback) noexcept override + // TODO: Implement + { unsupported("queryRealisation"); } +}; + +} From e43bb655feaa23977322d68278c88ac075eb2c41 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Sun, 10 Dec 2023 13:58:35 -0800 Subject: [PATCH 107/141] libstore/daemon.cc: note trust model difference in readDerivation()s Below the comment added by this commit is a much longer comment followed by a trust check, both of which have confused me on at least two occasions. I figured it out once, forgot it, then had to ask @Ericson2314 to explain it, at which point I understood it again. I think this might confuse other people too, or maybe I will just forget it a third time. So let's add a comment. Farther down in the function is the following check: ``` if (!(drvType.isCA() || trusted)) throw Error("you are not privileged to build input-addressed derivations"); ``` This seems really strange at first. A key property of Nix is that you can compute the outpath of a derivation using the derivation (and its references-closure) without trusting anybody! The missing insight is that at this point in the code the builder doesn't necessarily have the references-closure of the derivation being built, and therefore needs to trust that the derivation's outPath is honest. It's incredibly easy to overlook this, because the only difference between these two cases is which of these identically-named functions we used: - `readDerivation(Source,Store)` - `Store::readDerivation()` These functions have different trust models (except in the special case where the first function is used on the local store). We should call the reader's attention to this fact. Co-authored-by: Cole Helbling --- src/libstore/daemon.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 530b1a178..a112d6d31 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -574,6 +574,15 @@ static void performOp(TunnelLogger * logger, ref store, case WorkerProto::Op::BuildDerivation: { auto drvPath = store->parseStorePath(readString(from)); BasicDerivation drv; + /* + * Note: unlike wopEnsurePath, this operation reads a + * derivation-to-be-realized from the client with + * readDerivation(Source,Store) rather than reading it from + * the local store with Store::readDerivation(). Since the + * derivation-to-be-realized is not registered in the store + * it cannot be trusted that its outPath was calculated + * correctly. + */ readDerivation(from, *store, drv, Derivation::nameFromPath(drvPath)); BuildMode buildMode = (BuildMode) readInt(from); logger->startWork(); From 91ba7b230777e3fb023bda48c269d533702e50e8 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 6 Dec 2023 12:41:47 +0100 Subject: [PATCH 108/141] isAllowedURI: Extract function and test --- src/libexpr/eval.cc | 18 +++++-- src/libexpr/eval.hh | 5 ++ tests/unit/libexpr/eval.cc | 106 +++++++++++++++++++++++++++++++++++++ 3 files changed, 124 insertions(+), 5 deletions(-) create mode 100644 tests/unit/libexpr/eval.cc diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 9e494148e..0eb6f406e 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -599,21 +599,29 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & mkStorePathString(storePath, v); } -void EvalState::checkURI(const std::string & uri) +bool isAllowedURI(std::string_view uri, const Strings & allowedUris) { - if (!evalSettings.restrictEval) return; - /* 'uri' should be equal to a prefix, or in a subdirectory of a prefix. Thus, the prefix https://github.co does not permit access to https://github.com. Note: this allows 'http://' and 'https://' as prefixes for any http/https URI. */ - for (auto & prefix : evalSettings.allowedUris.get()) + for (auto & prefix : allowedUris) { if (uri == prefix || (uri.size() > prefix.size() && prefix.size() > 0 && hasPrefix(uri, prefix) && (prefix[prefix.size() - 1] == '/' || uri[prefix.size()] == '/'))) - return; + return true; + } + + return false; +} + +void EvalState::checkURI(const std::string & uri) +{ + if (!evalSettings.restrictEval) return; + + if (isAllowedURI(uri, evalSettings.allowedUris.get())) return; /* If the URI is a path, then check it against allowedPaths as well. */ diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index f3f6d35b9..6008c3f60 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -837,6 +837,11 @@ std::string showType(const Value & v); */ SourcePath resolveExprPath(SourcePath path); +/** + * Whether a URI is allowed, assuming restrictEval is enabled + */ +bool isAllowedURI(std::string_view uri, const Strings & allowedPaths); + struct InvalidPathError : EvalError { Path path; diff --git a/tests/unit/libexpr/eval.cc b/tests/unit/libexpr/eval.cc new file mode 100644 index 000000000..cc5d6bbfa --- /dev/null +++ b/tests/unit/libexpr/eval.cc @@ -0,0 +1,106 @@ +#include +#include + +#include "eval.hh" +#include "tests/libexpr.hh" + +namespace nix { + +TEST(nix_isAllowedURI, http_example_com) { + Strings allowed; + allowed.push_back("http://example.com"); + + ASSERT_TRUE(isAllowedURI("http://example.com", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com/foo", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com/foo/", allowed)); + ASSERT_FALSE(isAllowedURI("/", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.co", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.como", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.org", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.org/foo", allowed)); +} + +TEST(nix_isAllowedURI, http_example_com_foo) { + Strings allowed; + allowed.push_back("http://example.com/foo"); + + ASSERT_TRUE(isAllowedURI("http://example.com/foo", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com/foo/", allowed)); + ASSERT_FALSE(isAllowedURI("/foo", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.como", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.org/foo", allowed)); + // Broken? + // ASSERT_TRUE(isAllowedURI("http://example.com/foo?ok=1", allowed)); +} + +TEST(nix_isAllowedURI, http) { + Strings allowed; + allowed.push_back("http://"); + + ASSERT_TRUE(isAllowedURI("http://", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com/foo", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com/foo/", allowed)); + ASSERT_TRUE(isAllowedURI("http://example.com", allowed)); + ASSERT_FALSE(isAllowedURI("/", allowed)); + ASSERT_FALSE(isAllowedURI("https://", allowed)); + ASSERT_FALSE(isAllowedURI("http:foo", allowed)); +} + +TEST(nix_isAllowedURI, https) { + Strings allowed; + allowed.push_back("https://"); + + ASSERT_TRUE(isAllowedURI("https://example.com", allowed)); + ASSERT_TRUE(isAllowedURI("https://example.com/foo", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com/https:", allowed)); +} + +TEST(nix_isAllowedURI, absolute_path) { + Strings allowed; + allowed.push_back("/var/evil"); // bad idea + + ASSERT_TRUE(isAllowedURI("/var/evil", allowed)); + ASSERT_TRUE(isAllowedURI("/var/evil/", allowed)); + ASSERT_TRUE(isAllowedURI("/var/evil/foo", allowed)); + ASSERT_TRUE(isAllowedURI("/var/evil/foo/", allowed)); + ASSERT_FALSE(isAllowedURI("/", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evi", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evilo", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evilo/", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evilo/foo", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com/var/evil", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com//var/evil", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com//var/evil/foo", allowed)); +} + +TEST(nix_isAllowedURI, file_url) { + Strings allowed; + allowed.push_back("file:///var/evil"); // bad idea + + ASSERT_TRUE(isAllowedURI("file:///var/evil", allowed)); + ASSERT_TRUE(isAllowedURI("file:///var/evil/", allowed)); + ASSERT_TRUE(isAllowedURI("file:///var/evil/foo", allowed)); + ASSERT_TRUE(isAllowedURI("file:///var/evil/foo/", allowed)); + ASSERT_FALSE(isAllowedURI("/", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evi", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evilo", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evilo/", allowed)); + ASSERT_FALSE(isAllowedURI("/var/evilo/foo", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com/var/evil", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com//var/evil", allowed)); + ASSERT_FALSE(isAllowedURI("http://example.com//var/evil/foo", allowed)); + ASSERT_FALSE(isAllowedURI("http://var/evil", allowed)); + ASSERT_FALSE(isAllowedURI("http:///var/evil", allowed)); + ASSERT_FALSE(isAllowedURI("http://var/evil/", allowed)); + ASSERT_FALSE(isAllowedURI("file:///var/evi", allowed)); + ASSERT_FALSE(isAllowedURI("file:///var/evilo", allowed)); + ASSERT_FALSE(isAllowedURI("file:///var/evilo/", allowed)); + ASSERT_FALSE(isAllowedURI("file:///var/evilo/foo", allowed)); + ASSERT_FALSE(isAllowedURI("file:///", allowed)); + ASSERT_FALSE(isAllowedURI("file://", allowed)); +} + +} // namespace nix \ No newline at end of file From 6cbba914a70eb5da6447fee5528a63723ed13245 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 6 Dec 2023 12:43:20 +0100 Subject: [PATCH 109/141] isAllowedURI: Remove incorrect note --- src/libexpr/eval.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 0eb6f406e..d8a36fa02 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -603,8 +603,7 @@ bool isAllowedURI(std::string_view uri, const Strings & allowedUris) { /* 'uri' should be equal to a prefix, or in a subdirectory of a prefix. Thus, the prefix https://github.co does not permit - access to https://github.com. Note: this allows 'http://' and - 'https://' as prefixes for any http/https URI. */ + access to https://github.com. */ for (auto & prefix : allowedUris) { if (uri == prefix || (uri.size() > prefix.size() From 1fa958dda1ef0cb37441ef8d1a84faf6d501ac12 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 6 Dec 2023 14:08:22 +0100 Subject: [PATCH 110/141] isAllowedURI: Format --- src/libexpr/eval.cc | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index d8a36fa02..9e541f293 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -605,11 +605,14 @@ bool isAllowedURI(std::string_view uri, const Strings & allowedUris) prefix. Thus, the prefix https://github.co does not permit access to https://github.com. */ for (auto & prefix : allowedUris) { - if (uri == prefix || - (uri.size() > prefix.size() - && prefix.size() > 0 - && hasPrefix(uri, prefix) - && (prefix[prefix.size() - 1] == '/' || uri[prefix.size()] == '/'))) + if (uri == prefix + // Allow access to subdirectories of the prefix. + || (uri.size() > prefix.size() + && prefix.size() > 0 + && hasPrefix(uri, prefix) + && ( + prefix[prefix.size() - 1] == '/' + || uri[prefix.size()] == '/'))) return true; } From 79eb2920bb51c7ec9528a403986e79f04738e2be Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 6 Dec 2023 14:39:49 +0100 Subject: [PATCH 111/141] Add nix::isASCII*, locale-independent --- src/libutil/string.hh | 17 +++++++++++ tests/unit/libutil/string.cc | 59 ++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 src/libutil/string.hh create mode 100644 tests/unit/libutil/string.cc diff --git a/src/libutil/string.hh b/src/libutil/string.hh new file mode 100644 index 000000000..16ef75643 --- /dev/null +++ b/src/libutil/string.hh @@ -0,0 +1,17 @@ +#pragma once + +namespace nix { + + /** Locale-independent version of std::islower(). */ + inline bool isASCIILower(char c) { return c >= 'a' && c <= 'z'; }; + + /** Locale-independent version of std::isupper(). */ + inline bool isASCIIUpper(char c) { return c >= 'A' && c <= 'Z'; }; + + /** Locale-independent version of std::isalpha(). */ + inline bool isASCIIAlpha(char c) { return isASCIILower(c) || isASCIIUpper(c); }; + + /** Locale-independent version of std::isdigit(). */ + inline bool isASCIIDigit(char c) { return c >= '0' && c <= '9'; }; + +} diff --git a/tests/unit/libutil/string.cc b/tests/unit/libutil/string.cc new file mode 100644 index 000000000..381f2cc15 --- /dev/null +++ b/tests/unit/libutil/string.cc @@ -0,0 +1,59 @@ +#include +#include "string.hh" + +namespace nix { + +TEST(string, isASCIILower) { + ASSERT_TRUE(isASCIILower('a')); + ASSERT_TRUE(isASCIILower('z')); + ASSERT_FALSE(isASCIILower('A')); + ASSERT_FALSE(isASCIILower('Z')); + ASSERT_FALSE(isASCIILower('0')); + ASSERT_FALSE(isASCIILower('9')); + ASSERT_FALSE(isASCIILower(' ')); + ASSERT_FALSE(isASCIILower('\n')); + ASSERT_FALSE(isASCIILower('\t')); + ASSERT_FALSE(isASCIILower(':')); +} + +TEST(string, isASCIIUpper) { + ASSERT_FALSE(isASCIIUpper('a')); + ASSERT_FALSE(isASCIIUpper('z')); + ASSERT_TRUE(isASCIIUpper('A')); + ASSERT_TRUE(isASCIIUpper('Z')); + ASSERT_FALSE(isASCIIUpper('0')); + ASSERT_FALSE(isASCIIUpper('9')); + ASSERT_FALSE(isASCIIUpper(' ')); + ASSERT_FALSE(isASCIIUpper('\n')); + ASSERT_FALSE(isASCIIUpper('\t')); + ASSERT_FALSE(isASCIIUpper(':')); +} + +TEST(string, isASCIIAlpha) { + ASSERT_TRUE(isASCIIAlpha('a')); + ASSERT_TRUE(isASCIIAlpha('z')); + ASSERT_TRUE(isASCIIAlpha('A')); + ASSERT_TRUE(isASCIIAlpha('Z')); + ASSERT_FALSE(isASCIIAlpha('0')); + ASSERT_FALSE(isASCIIAlpha('9')); + ASSERT_FALSE(isASCIIAlpha(' ')); + ASSERT_FALSE(isASCIIAlpha('\n')); + ASSERT_FALSE(isASCIIAlpha('\t')); + ASSERT_FALSE(isASCIIAlpha(':')); +} + +TEST(string, isASCIIDigit) { + ASSERT_FALSE(isASCIIDigit('a')); + ASSERT_FALSE(isASCIIDigit('z')); + ASSERT_FALSE(isASCIIDigit('A')); + ASSERT_FALSE(isASCIIDigit('Z')); + ASSERT_TRUE(isASCIIDigit('0')); + ASSERT_TRUE(isASCIIDigit('1')); + ASSERT_TRUE(isASCIIDigit('9')); + ASSERT_FALSE(isASCIIDigit(' ')); + ASSERT_FALSE(isASCIIDigit('\n')); + ASSERT_FALSE(isASCIIDigit('\t')); + ASSERT_FALSE(isASCIIDigit(':')); +} + +} \ No newline at end of file From d3a85b68347071d8d93ec796a38c707483d7b272 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 6 Dec 2023 15:14:41 +0100 Subject: [PATCH 112/141] isValidSchemeName: Add function --- src/libutil/url.cc | 17 +++++++++++++++++ src/libutil/url.hh | 9 +++++++++ tests/unit/libutil/url.cc | 18 ++++++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 57b64d607..f2d5f1782 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -3,6 +3,7 @@ #include "util.hh" #include "split.hh" #include "canon-path.hh" +#include "string.hh" namespace nix { @@ -183,4 +184,20 @@ std::string fixGitURL(const std::string & url) } } +// https://www.rfc-editor.org/rfc/rfc3986#section-3.1 +bool isValidSchemeName(std::string_view s) +{ + if (s.empty()) return false; + if (!isASCIIAlpha(s[0])) return false; + for (auto c : s.substr(1)) { + if (isASCIIAlpha(c)) continue; + if (isASCIIDigit(c)) continue; + if (c == '+') continue; + if (c == '-') continue; + if (c == '.') continue; + return false; + } + return true; +} + } diff --git a/src/libutil/url.hh b/src/libutil/url.hh index 833f54678..24806bbff 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -55,4 +55,13 @@ ParsedUrlScheme parseUrlScheme(std::string_view scheme); changes absolute paths into file:// URLs. */ std::string fixGitURL(const std::string & url); +/** + * Whether a string is valid as RFC 3986 scheme name. + * Colon `:` is part of the URI; not the scheme name, and therefore rejected. + * See https://www.rfc-editor.org/rfc/rfc3986#section-3.1 + * + * Does not check whether the scheme is understood, as that's context-dependent. + */ +bool isValidSchemeName(std::string_view scheme); + } diff --git a/tests/unit/libutil/url.cc b/tests/unit/libutil/url.cc index a678dad20..09fa4e218 100644 --- a/tests/unit/libutil/url.cc +++ b/tests/unit/libutil/url.cc @@ -344,4 +344,22 @@ namespace nix { ASSERT_EQ(percentDecode(e), s); } +TEST(nix, isValidSchemeName) { + ASSERT_TRUE(isValidSchemeName("http")); + ASSERT_TRUE(isValidSchemeName("https")); + ASSERT_TRUE(isValidSchemeName("file")); + ASSERT_TRUE(isValidSchemeName("file+https")); + ASSERT_TRUE(isValidSchemeName("fi.le")); + ASSERT_TRUE(isValidSchemeName("file-ssh")); + ASSERT_TRUE(isValidSchemeName("file+")); + ASSERT_TRUE(isValidSchemeName("file.")); + ASSERT_TRUE(isValidSchemeName("file1")); + ASSERT_FALSE(isValidSchemeName("file:")); + ASSERT_FALSE(isValidSchemeName("file/")); + ASSERT_FALSE(isValidSchemeName("+file")); + ASSERT_FALSE(isValidSchemeName(".file")); + ASSERT_FALSE(isValidSchemeName("-file")); + ASSERT_FALSE(isValidSchemeName("1file")); +} + } From a05bc9eb92371af631fc9fb83c3595957fb56943 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 6 Dec 2023 15:27:29 +0100 Subject: [PATCH 113/141] allowed-uris: Match whole schemes also when scheme is not followed by slashes --- ...llowed-uris-can-now-match-whole-schemes.md | 7 ++++ src/libexpr/eval-settings.hh | 5 +++ src/libexpr/eval.cc | 17 ++++++++- tests/unit/libexpr/eval.cc | 35 +++++++++++++++++++ 4 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 doc/manual/rl-next/allowed-uris-can-now-match-whole-schemes.md diff --git a/doc/manual/rl-next/allowed-uris-can-now-match-whole-schemes.md b/doc/manual/rl-next/allowed-uris-can-now-match-whole-schemes.md new file mode 100644 index 000000000..3cf75a612 --- /dev/null +++ b/doc/manual/rl-next/allowed-uris-can-now-match-whole-schemes.md @@ -0,0 +1,7 @@ +--- +synopsis: Option `allowed-uris` can now match whole schemes in URIs without slashes +prs: 9547 +--- + +If a scheme, such as `github:` is specified in the `allowed-uris` option, all URIs starting with `github:` are allowed. +Previously this only worked for schemes whose URIs used the `://` syntax. diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index db2971acb..3009a462c 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -68,6 +68,11 @@ struct EvalSettings : Config evaluation mode. For example, when set to `https://github.com/NixOS`, builtin functions such as `fetchGit` are allowed to access `https://github.com/NixOS/patchelf.git`. + + Access is granted when + - the URI is equal to the prefix, + - or the URI is a subpath of the prefix, + - or the prefix is a URI scheme ended by a colon `:` and the URI has the same scheme. )"}; Setting traceFunctionCalls{this, false, "trace-function-calls", diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 9e541f293..1552e3e92 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -18,6 +18,7 @@ #include "memory-input-accessor.hh" #include "signals.hh" #include "gc-small-vector.hh" +#include "url.hh" #include #include @@ -599,6 +600,14 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & mkStorePathString(storePath, v); } +inline static bool isJustSchemePrefix(std::string_view prefix) +{ + return + !prefix.empty() + && prefix[prefix.size() - 1] == ':' + && isValidSchemeName(prefix.substr(0, prefix.size() - 1)); +} + bool isAllowedURI(std::string_view uri, const Strings & allowedUris) { /* 'uri' should be equal to a prefix, or in a subdirectory of a @@ -611,8 +620,14 @@ bool isAllowedURI(std::string_view uri, const Strings & allowedUris) && prefix.size() > 0 && hasPrefix(uri, prefix) && ( + // Allow access to subdirectories of the prefix. prefix[prefix.size() - 1] == '/' - || uri[prefix.size()] == '/'))) + || uri[prefix.size()] == '/' + + // Allow access to whole schemes + || isJustSchemePrefix(prefix) + ) + )) return true; } diff --git a/tests/unit/libexpr/eval.cc b/tests/unit/libexpr/eval.cc index cc5d6bbfa..93d3f658f 100644 --- a/tests/unit/libexpr/eval.cc +++ b/tests/unit/libexpr/eval.cc @@ -103,4 +103,39 @@ TEST(nix_isAllowedURI, file_url) { ASSERT_FALSE(isAllowedURI("file://", allowed)); } +TEST(nix_isAllowedURI, github_all) { + Strings allowed; + allowed.push_back("github:"); + ASSERT_TRUE(isAllowedURI("github:", allowed)); + ASSERT_TRUE(isAllowedURI("github:foo/bar", allowed)); + ASSERT_TRUE(isAllowedURI("github:foo/bar/feat-multi-bar", allowed)); + ASSERT_TRUE(isAllowedURI("github:foo/bar?ref=refs/heads/feat-multi-bar", allowed)); + ASSERT_TRUE(isAllowedURI("github://foo/bar", allowed)); + ASSERT_FALSE(isAllowedURI("https://github:443/foo/bar/archive/master.tar.gz", allowed)); + ASSERT_FALSE(isAllowedURI("file://github:foo/bar/archive/master.tar.gz", allowed)); + ASSERT_FALSE(isAllowedURI("file:///github:foo/bar/archive/master.tar.gz", allowed)); + ASSERT_FALSE(isAllowedURI("github", allowed)); +} + +TEST(nix_isAllowedURI, github_org) { + Strings allowed; + allowed.push_back("github:foo"); + ASSERT_FALSE(isAllowedURI("github:", allowed)); + ASSERT_TRUE(isAllowedURI("github:foo/bar", allowed)); + ASSERT_TRUE(isAllowedURI("github:foo/bar/feat-multi-bar", allowed)); + ASSERT_TRUE(isAllowedURI("github:foo/bar?ref=refs/heads/feat-multi-bar", allowed)); + ASSERT_FALSE(isAllowedURI("github://foo/bar", allowed)); + ASSERT_FALSE(isAllowedURI("https://github:443/foo/bar/archive/master.tar.gz", allowed)); + ASSERT_FALSE(isAllowedURI("file://github:foo/bar/archive/master.tar.gz", allowed)); + ASSERT_FALSE(isAllowedURI("file:///github:foo/bar/archive/master.tar.gz", allowed)); +} + +TEST(nix_isAllowedURI, non_scheme_colon) { + Strings allowed; + allowed.push_back("https://foo/bar:"); + ASSERT_TRUE(isAllowedURI("https://foo/bar:", allowed)); + ASSERT_TRUE(isAllowedURI("https://foo/bar:/baz", allowed)); + ASSERT_FALSE(isAllowedURI("https://foo/bar:baz", allowed)); +} + } // namespace nix \ No newline at end of file From 89cf53648ca98434a40b0c0cef51fa64f6e0fa37 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 11 Dec 2023 12:26:31 +0100 Subject: [PATCH 114/141] Contributing branches and reverting (#9577) Co-authored-by: Valentin Gagarin --- doc/manual/src/contributing/hacking.md | 42 ++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 237eff925..4d3d66397 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -282,3 +282,45 @@ See also the [format documentation](https://github.com/haskell/cabal/blob/master Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. Set `buildUnreleasedNotes = true;` in `flake.nix` to build the release notes on the fly. + +## Branches + +- [`master`](https://github.com/NixOS/nix/commits/master) + + The main development branch. All changes are approved and merged here. + When developing a change, create a branch based on the latest `master`. + + Maintainers try to [keep it in a release-worthy state](#reverting). + +- [`maintenance-*.*`](https://github.com/NixOS/nix/branches/all?query=maintenance) + + These branches are the subject of backports only, and are + also [kept](#reverting) in a release-worthy state. + + See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) + +- [`latest-release`](https://github.com/NixOS/nix/tree/latest-release) + + The latest patch release of the latest minor version. + + See [`maintainers/release-process.md`](https://github.com/NixOS/nix/blob/master/maintainers/release-process.md) + +- [`backport-*-to-*`](https://github.com/NixOS/nix/branches/all?query=backport) + + Generally branches created by the backport action. + + See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) + +- [_other_](https://github.com/NixOS/nix/branches/all) + + Branches that do not conform to the above patterns should be feature branches. + +## Reverting + +If a change turns out to be merged by mistake, or contain a regression, it may be reverted. +A revert is not a rejection of the contribution, but merely part of an effective development process. +It makes sure that development keeps running smoothly, with minimal uncertainty, and less overhead. +If maintainers have to worry too much about avoiding reverts, they would not be able to merge as much. +By embracing reverts as a good part of the development process, everyone wins. + +However, taking a step back may be frustrating, so maintainers will be extra supportive on the next try. From f45d2ee2b7090560fc30a227d638684268af700d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20M=C3=B6ller?= Date: Mon, 11 Dec 2023 16:02:09 +0100 Subject: [PATCH 115/141] Fix query parsing for path-like flakes --- src/libexpr/flake/flakeref.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 16f45ace7..8b0eb7460 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -90,7 +90,7 @@ std::pair parsePathFlakeRefWithFragment( fragment = percentDecode(url.substr(fragmentStart+1)); } if (pathEnd != std::string::npos && fragmentStart != std::string::npos) { - query = decodeQuery(url.substr(pathEnd+1, fragmentStart)); + query = decodeQuery(url.substr(pathEnd+1, fragmentStart-pathEnd-1)); } if (baseDir) { From 994f1b5c0de44319992ef6b1b106cee3fa400dc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20M=C3=B6ller?= Date: Mon, 11 Dec 2023 16:05:34 +0100 Subject: [PATCH 116/141] Add test cases for flake urls with fragments --- tests/functional/flakes/flakes.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index ccf1699f9..7506b6b3b 100644 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -193,6 +193,14 @@ nix build -o "$TEST_ROOT/result" flake1 nix build -o "$TEST_ROOT/result" "$flake1Dir" nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir" +# Test explicit packages.default. +nix build -o "$TEST_ROOT/result" "$flake1Dir#default" +nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir#default" + +# Test explicit packages.default with query. +nix build -o "$TEST_ROOT/result" "$flake1Dir?ref=HEAD#default" +nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir?ref=HEAD#default" + # Check that store symlinks inside a flake are not interpreted as flakes. nix build -o "$flake1Dir/result" "git+file://$flake1Dir" nix path-info "$flake1Dir/result" From 5f30c8acc7e0cad08924cc53e350e811d097fae7 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 10 Dec 2023 18:51:23 -0500 Subject: [PATCH 117/141] Give `Store::queryDerivationOutputMap` and `evalStore` argument Picking up where https://github.com/NixOS/nix/pull/9563 left off. --- src/libstore/store-api.cc | 4 ++-- src/libstore/store-api.hh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 800df7fa0..7f35e74af 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -547,8 +547,8 @@ std::map> Store::queryPartialDerivationOut return outputs; } -OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) { - auto resp = queryPartialDerivationOutputMap(path); +OutputPathMap Store::queryDerivationOutputMap(const StorePath & path, Store * evalStore) { + auto resp = queryPartialDerivationOutputMap(path, evalStore); OutputPathMap result; for (auto & [outName, optOutPath] : resp) { if (!optOutPath) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index ada6699d5..13e5a1446 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -369,7 +369,7 @@ public: * Query the mapping outputName=>outputPath for the given derivation. * Assume every output has a mapping and throw an exception otherwise. */ - OutputPathMap queryDerivationOutputMap(const StorePath & path); + OutputPathMap queryDerivationOutputMap(const StorePath & path, Store * evalStore = nullptr); /** * Query the full store path given the hash part of a valid store From 9f39dda66ce0f92707d4be05d0a90961c78f8bd4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 10 Dec 2023 21:21:21 -0500 Subject: [PATCH 118/141] Fix building CA derivations with and eval store I don't love the way this code looks. There are two larger problems: - eval, build/scratch, destination stores (#5025) should have different types to reflect the fact that they are used for different purposes and those purposes correspond to different operations. It should be impossible to "use the wrong store" in my cases. - Since drvs can end up in both the eval and build/scratch store, we should have some sort of union/layered store (not on the file sytem level, just conceptual level) that allows accessing both. This would get rid of the ugly "check both" boilerplate in this PR. Still, it might be better to land this now / soon after minimal cleanup, so we have a concrete idea of what problem better abstractions are supposed to solve. --- src/libstore/build/derivation-goal.cc | 51 +++++++++++++++++++++------ src/libstore/misc.cc | 9 +++-- src/libstore/store-api.hh | 3 +- src/nix-build/nix-build.cc | 6 ++-- tests/functional/ca/eval-store.sh | 10 ++++++ tests/functional/ca/local.mk | 1 + tests/functional/eval-store.sh | 16 +++++++-- 7 files changed, 76 insertions(+), 20 deletions(-) create mode 100644 tests/functional/ca/eval-store.sh diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index d4da374ba..f8728ed4a 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -196,10 +196,19 @@ void DerivationGoal::loadDerivation() things being garbage collected while we're busy. */ worker.evalStore.addTempRoot(drvPath); - assert(worker.evalStore.isValidPath(drvPath)); + /* Get the derivation. It is probably in the eval store, but it might be inthe main store: - /* Get the derivation. */ - drv = std::make_unique(worker.evalStore.readDerivation(drvPath)); + - Resolved derivation are resolved against main store realisations, and so must be stored there. + + - Dynamic derivations are built, and so are found in the main store. + */ + for (auto * drvStore : { &worker.evalStore, &worker.store }) { + if (drvStore->isValidPath(drvPath)) { + drv = std::make_unique(drvStore->readDerivation(drvPath)); + break; + } + } + assert(drv); haveDerivation(); } @@ -401,11 +410,15 @@ void DerivationGoal::gaveUpOnSubstitution() } /* Copy the input sources from the eval store to the build - store. */ + store. + + Note that some inputs might not be in the eval store because they + are (resolved) derivation outputs in a resolved derivation. */ if (&worker.evalStore != &worker.store) { RealisedPath::Set inputSrcs; for (auto & i : drv->inputSrcs) - inputSrcs.insert(i); + if (worker.evalStore.isValidPath(i)) + inputSrcs.insert(i); copyClosure(worker.evalStore, worker.store, inputSrcs); } @@ -453,7 +466,7 @@ void DerivationGoal::repairClosure() std::map outputsToDrv; for (auto & i : inputClosure) if (i.isDerivation()) { - auto depOutputs = worker.store.queryPartialDerivationOutputMap(i); + auto depOutputs = worker.store.queryPartialDerivationOutputMap(i, &worker.evalStore); for (auto & j : depOutputs) if (j.second) outputsToDrv.insert_or_assign(*j.second, i); @@ -604,7 +617,13 @@ void DerivationGoal::inputsRealised() return *outPath; } else { - auto outMap = worker.evalStore.queryDerivationOutputMap(depDrvPath); + auto outMap = [&]{ + for (auto * drvStore : { &worker.evalStore, &worker.store }) + if (drvStore->isValidPath(depDrvPath)) + return worker.store.queryDerivationOutputMap(depDrvPath, drvStore); + assert(false); + }(); + auto outMapPath = outMap.find(outputName); if (outMapPath == outMap.end()) { throw Error( @@ -1085,8 +1104,12 @@ void DerivationGoal::resolvedFinished() auto newRealisation = realisation; newRealisation.id = DrvOutput { initialOutput->outputHash, outputName }; newRealisation.signatures.clear(); - if (!drv->type().isFixed()) - newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath); + if (!drv->type().isFixed()) { + auto & drvStore = worker.evalStore.isValidPath(drvPath) + ? worker.evalStore + : worker.store; + newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation.outPath, &drvStore); + } signRealisation(newRealisation); worker.store.registerDrvOutput(newRealisation); } @@ -1379,7 +1402,10 @@ std::map> DerivationGoal::queryPartialDeri res.insert_or_assign(name, output.path(worker.store, drv->name, name)); return res; } else { - return worker.store.queryPartialDerivationOutputMap(drvPath); + for (auto * drvStore : { &worker.evalStore, &worker.store }) + if (drvStore->isValidPath(drvPath)) + return worker.store.queryPartialDerivationOutputMap(drvPath, drvStore); + assert(false); } } @@ -1392,7 +1418,10 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap() res.insert_or_assign(name, *output.second); return res; } else { - return worker.store.queryDerivationOutputMap(drvPath); + for (auto * drvStore : { &worker.evalStore, &worker.store }) + if (drvStore->isValidPath(drvPath)) + return worker.store.queryDerivationOutputMap(drvPath, drvStore); + assert(false); } } diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 9f63fbbb5..cc8ad3d02 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -331,8 +331,11 @@ std::map drvOutputReferences( std::map drvOutputReferences( Store & store, const Derivation & drv, - const StorePath & outputPath) + const StorePath & outputPath, + Store * evalStore_) { + auto & evalStore = evalStore_ ? *evalStore_ : store; + std::set inputRealisations; std::function::ChildNode &)> accumRealisations; @@ -340,7 +343,7 @@ std::map drvOutputReferences( accumRealisations = [&](const StorePath & inputDrv, const DerivedPathMap::ChildNode & inputNode) { if (!inputNode.value.empty()) { auto outputHashes = - staticOutputHashes(store, store.readDerivation(inputDrv)); + staticOutputHashes(evalStore, evalStore.readDerivation(inputDrv)); for (const auto & outputName : inputNode.value) { auto outputHash = get(outputHashes, outputName); if (!outputHash) @@ -362,7 +365,7 @@ std::map drvOutputReferences( SingleDerivedPath next = SingleDerivedPath::Built { d, outputName }; accumRealisations( // TODO deep resolutions for dynamic derivations, issue #8947, would go here. - resolveDerivedPath(store, next), + resolveDerivedPath(store, next, evalStore_), childNode); } } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 13e5a1446..2c883ce97 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -943,6 +943,7 @@ const ContentAddress * getDerivationCA(const BasicDerivation & drv); std::map drvOutputReferences( Store & store, const Derivation & drv, - const StorePath & outputPath); + const StorePath & outputPath, + Store * evalStore = nullptr); } diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 01da028d8..8e9be14c1 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -462,7 +462,7 @@ static void main_nix_build(int argc, char * * argv) if (dryRun) return; if (shellDrv) { - auto shellDrvOutputs = store->queryPartialDerivationOutputMap(shellDrv.value()); + auto shellDrvOutputs = store->queryPartialDerivationOutputMap(shellDrv.value(), &*evalStore); shell = store->printStorePath(shellDrvOutputs.at("out").value()) + "/bin/bash"; } @@ -515,7 +515,7 @@ static void main_nix_build(int argc, char * * argv) std::function::ChildNode &)> accumInputClosure; accumInputClosure = [&](const StorePath & inputDrv, const DerivedPathMap::ChildNode & inputNode) { - auto outputs = evalStore->queryPartialDerivationOutputMap(inputDrv); + auto outputs = store->queryPartialDerivationOutputMap(inputDrv, &*evalStore); for (auto & i : inputNode.value) { auto o = outputs.at(i); store->computeFSClosure(*o, inputs); @@ -653,7 +653,7 @@ static void main_nix_build(int argc, char * * argv) if (counter) drvPrefix += fmt("-%d", counter + 1); - auto builtOutputs = evalStore->queryPartialDerivationOutputMap(drvPath); + auto builtOutputs = store->queryPartialDerivationOutputMap(drvPath, &*evalStore); auto maybeOutputPath = builtOutputs.at(outputName); assert(maybeOutputPath); diff --git a/tests/functional/ca/eval-store.sh b/tests/functional/ca/eval-store.sh new file mode 100644 index 000000000..9cc499606 --- /dev/null +++ b/tests/functional/ca/eval-store.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +# Ensure that garbage collection works properly with ca derivations + +source common.sh + +export NIX_TESTS_CA_BY_DEFAULT=1 + +cd .. +source eval-store.sh diff --git a/tests/functional/ca/local.mk b/tests/functional/ca/local.mk index fd87b8d1f..4f86b268f 100644 --- a/tests/functional/ca/local.mk +++ b/tests/functional/ca/local.mk @@ -5,6 +5,7 @@ ca-tests := \ $(d)/concurrent-builds.sh \ $(d)/derivation-json.sh \ $(d)/duplicate-realisation-in-closure.sh \ + $(d)/eval-store.sh \ $(d)/gc.sh \ $(d)/import-derivation.sh \ $(d)/new-build-cmd.sh \ diff --git a/tests/functional/eval-store.sh b/tests/functional/eval-store.sh index 8fc859730..ec99fd953 100644 --- a/tests/functional/eval-store.sh +++ b/tests/functional/eval-store.sh @@ -11,7 +11,16 @@ rm -rf "$eval_store" nix build -f dependencies.nix --eval-store "$eval_store" -o "$TEST_ROOT/result" [[ -e $TEST_ROOT/result/foobar ]] -(! ls $NIX_STORE_DIR/*.drv) +if [[ ! -n "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then + # Resolved CA derivations are written to store for building + # + # TODO when we something more systematic + # (https://github.com/NixOS/nix/issues/5025) that distinguishes + # between scratch storage for building and the final destination + # store, we'll be able to make this unconditional again -- resolved + # derivations should only appear in the scratch store. + (! ls $NIX_STORE_DIR/*.drv) +fi ls $eval_store/nix/store/*.drv clearStore @@ -26,5 +35,8 @@ rm -rf "$eval_store" nix-build dependencies.nix --eval-store "$eval_store" -o "$TEST_ROOT/result" [[ -e $TEST_ROOT/result/foobar ]] -(! ls $NIX_STORE_DIR/*.drv) +if [[ ! -n "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then + # See above + (! ls $NIX_STORE_DIR/*.drv) +fi ls $eval_store/nix/store/*.drv From 0b81557e2cf30cebb916f82f192f04df38c810d7 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 6 Dec 2023 20:14:14 -0500 Subject: [PATCH 119/141] flake.nix: Put some list items on their own line These things are about to become longer --- flake.nix | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/flake.nix b/flake.nix index 99480183a..973bb55af 100644 --- a/flake.nix +++ b/flake.nix @@ -36,8 +36,10 @@ systems = linuxSystems ++ darwinSystems; crossSystems = [ - "armv6l-linux" "armv7l-linux" - "x86_64-freebsd13" "x86_64-netbsd" + "armv6l-linux" + "armv7l-linux" + "x86_64-freebsd13" + "x86_64-netbsd" ]; stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" "libcxxStdenv" "ccacheStdenv" ]; @@ -575,8 +577,25 @@ # to https://nixos.org/nix/install. It downloads the binary # tarball for the user's system and calls the second half of the # installation script. - installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" "armv6l-linux" "armv7l-linux" ]; - installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" "armv6l-linux" "armv7l-linux"]; + installerScript = installScriptFor [ + # Native + "x86_64-linux" + "i686-linux" + "aarch64-linux" + "x86_64-darwin" + "aarch64-darwin" + # Cross + "armv6l-linux" + "armv7l-linux" + ]; + installerScriptForGHA = installScriptFor [ + # Native + "x86_64-linux" + "x86_64-darwin" + # Cross + "armv6l-linux" + "armv7l-linux" + ]; # docker image with Nix inside dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); From f60c2e8a5acabf5fb554f77014904bb0d0c91604 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 6 Dec 2023 20:28:11 -0500 Subject: [PATCH 120/141] flake.nix: `installScriptFor` take tarballs not strings Trying to look up keys in multiple places is not nice, better for the caller to be explicit. --- flake.nix | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/flake.nix b/flake.nix index 973bb55af..185140092 100644 --- a/flake.nix +++ b/flake.nix @@ -256,7 +256,7 @@ ]; }; - installScriptFor = systems: + installScriptFor = tarballs: with nixpkgsFor.x86_64-linux.native; runCommand "installer-script" { buildInputs = [ nix ]; @@ -277,14 +277,14 @@ substitute ${./scripts/install.in} $out/install \ ${pkgs.lib.concatMapStrings - (system: let - tarball = if builtins.elem system crossSystems then self.hydraJobs.binaryTarballCross.x86_64-linux.${system} else self.hydraJobs.binaryTarball.${system}; + (tarball: let + inherit (tarball.stdenv.hostPlatform) system; in '' \ --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${tarball}/*.tar.xz) \ --replace '@tarballPath_${system}@' $(tarballPath ${tarball}/*.tar.xz) \ '' ) - systems + tarballs } --replace '@nixVersion@' ${version} echo "file installer $out/install" >> $out/nix-support/hydra-build-products @@ -341,7 +341,7 @@ installerClosureInfo = buildPackages.closureInfo { rootPaths = [ nix cacert ]; }; in - buildPackages.runCommand "nix-binary-tarball-${version}" + pkgs.runCommand "nix-binary-tarball-${version}" { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck; meta.description = "Distribution-independent Nix bootstrap binaries for ${pkgs.system}"; } @@ -579,22 +579,22 @@ # installation script. installerScript = installScriptFor [ # Native - "x86_64-linux" - "i686-linux" - "aarch64-linux" - "x86_64-darwin" - "aarch64-darwin" + self.hydraJobs.binaryTarball."x86_64-linux" + self.hydraJobs.binaryTarball."i686-linux" + self.hydraJobs.binaryTarball."aarch64-linux" + self.hydraJobs.binaryTarball."x86_64-darwin" + self.hydraJobs.binaryTarball."aarch64-darwin" # Cross - "armv6l-linux" - "armv7l-linux" + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-linux" + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-linux" ]; installerScriptForGHA = installScriptFor [ # Native - "x86_64-linux" - "x86_64-darwin" + self.hydraJobs.binaryTarball."x86_64-linux" + self.hydraJobs.binaryTarball."x86_64-darwin" # Cross - "armv6l-linux" - "armv7l-linux" + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-linux" + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-linux" ]; # docker image with Nix inside From 78492cfde73d57ca01c73d77a23440754c9e7ee4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 2 Sep 2023 14:36:25 -0400 Subject: [PATCH 121/141] flake.nix: Use `config` not `system` for cross so we can be a bit more precise --- flake.nix | 20 ++++++++++---------- maintainers/upload-release.pl | 4 ++-- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/flake.nix b/flake.nix index 185140092..2f6668b81 100644 --- a/flake.nix +++ b/flake.nix @@ -36,10 +36,10 @@ systems = linuxSystems ++ darwinSystems; crossSystems = [ - "armv6l-linux" - "armv7l-linux" - "x86_64-freebsd13" - "x86_64-netbsd" + "armv6l-unknown-linux-gnueabihf" + "armv7l-unknown-linux-gnueabihf" + "x86_64-unknown-freebsd13" + "x86_64-unknown-netbsd" ]; stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" "libcxxStdenv" "ccacheStdenv" ]; @@ -116,8 +116,8 @@ inherit system; }; crossSystem = if crossSystem == null then null else { - system = crossSystem; - } // lib.optionalAttrs (crossSystem == "x86_64-freebsd13") { + config = crossSystem; + } // lib.optionalAttrs (crossSystem == "x86_64-unknown-freebsd13") { useLLVM = true; }; overlays = [ @@ -585,16 +585,16 @@ self.hydraJobs.binaryTarball."x86_64-darwin" self.hydraJobs.binaryTarball."aarch64-darwin" # Cross - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-linux" - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-linux" + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" ]; installerScriptForGHA = installScriptFor [ # Native self.hydraJobs.binaryTarball."x86_64-linux" self.hydraJobs.binaryTarball."x86_64-darwin" # Cross - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-linux" - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-linux" + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" ]; # docker image with Nix inside diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index ebc536f12..4e2c379f0 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -154,8 +154,8 @@ downloadFile("binaryTarball.x86_64-linux", "1"); downloadFile("binaryTarball.aarch64-linux", "1"); downloadFile("binaryTarball.x86_64-darwin", "1"); downloadFile("binaryTarball.aarch64-darwin", "1"); -downloadFile("binaryTarballCross.x86_64-linux.armv6l-linux", "1"); -downloadFile("binaryTarballCross.x86_64-linux.armv7l-linux", "1"); +downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1"); +downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1"); downloadFile("installerScript", "1"); # Upload docker images to dockerhub. From 46b98a40a7c5488a99525bc780b7f7bba0131545 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 2 Sep 2023 14:38:33 -0400 Subject: [PATCH 122/141] flake.nix: Make changes so a MinGW dev shell would work --- flake.nix | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/flake.nix b/flake.nix index 2f6668b81..f0dd9a3e3 100644 --- a/flake.nix +++ b/flake.nix @@ -210,7 +210,7 @@ buildDeps = [ curl - bzip2 xz brotli editline + bzip2 xz brotli openssl sqlite libarchive (pkgs.libgit2.overrideAttrs (attrs: { @@ -219,10 +219,13 @@ cmakeFlags = (attrs.cmakeFlags or []) ++ ["-DUSE_SSH=exec"]; })) boost - lowdown-nix libsodium ] - ++ lib.optionals stdenv.isLinux [libseccomp] + ++ lib.optionals (!stdenv.hostPlatform.isWindows) [ + editline + lowdown-nix + ] + ++ lib.optional stdenv.isLinux libseccomp ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid; checkDeps = [ @@ -510,7 +513,7 @@ stdenv = currentStdenv; }; - meta.platforms = lib.platforms.unix; + meta.platforms = lib.platforms.unix ++ lib.platforms.windows; meta.mainProgram = "nix"; }); From b892161e314d976e7692ffcf487e1aa042165745 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 11 Dec 2023 12:26:42 -0500 Subject: [PATCH 123/141] flake.nix: Make a MinGW dev shell This requires a `shellCrossSystems` for now, since Nix doesn't actually build on Windows. This can be dropped once it does. --- flake.nix | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index f0dd9a3e3..bbdfc38e9 100644 --- a/flake.nix +++ b/flake.nix @@ -42,6 +42,13 @@ "x86_64-unknown-netbsd" ]; + # Nix doesn't yet build on this platform, so we put it in a + # separate list. We just use this for `devShells` and + # `nixpkgsFor`, which this depends on. + shellCrossSystems = crossSystems ++ [ + "x86_64-w64-mingw32" + ]; + stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" "libcxxStdenv" "ccacheStdenv" ]; forAllSystems = lib.genAttrs systems; @@ -129,7 +136,7 @@ in { inherit stdenvs native; static = native.pkgsStatic; - cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv"); + cross = lib.genAttrs shellCrossSystems (crossSystem: make-pkgs crossSystem "stdenv"); }); commonDeps = @@ -808,7 +815,7 @@ in (makeShells "native" nixpkgsFor.${system}.native) // (makeShells "static" nixpkgsFor.${system}.static) // - (forAllCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) // + (lib.genAttrs shellCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) // { default = self.devShells.${system}.native-stdenvPackages; } From 589fb105f311af65230d374cbbddf7173c7ad103 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Tue, 12 Dec 2023 16:05:32 +0100 Subject: [PATCH 124/141] Fix the VM tests Work around https://github.com/NixOS/nixpkgs/issues/271146 until we can depend on a Nixpkgs version containing https://github.com/NixOS/nixpkgs/pull/271423 --- flake.nix | 4 ++++ tests/nixos/default.nix | 1 + 2 files changed, 5 insertions(+) diff --git a/flake.nix b/flake.nix index bbdfc38e9..ada52c05d 100644 --- a/flake.nix +++ b/flake.nix @@ -7,6 +7,10 @@ # Also, do not grab arbitrary further staging commits. This PR was # carefully made to be based on release-23.05 and just contain # rebuild-causing changes to packages that Nix actually uses. + # + # Once this is updated to something containing + # https://github.com/NixOS/nixpkgs/pull/271423, don't forget + # to remove the `nix.checkAllErrors = false;` line in the tests. inputs.nixpkgs.url = "github:NixOS/nixpkgs/staging-23.05"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index 4459aa664..2645cac8e 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -10,6 +10,7 @@ let hostPkgs = nixpkgsFor.${system}.native; defaults = { nixpkgs.pkgs = nixpkgsFor.${system}.native; + nix.checkAllErrors = false; }; _module.args.nixpkgs = nixpkgs; }; From 2e451a663eff96b89360cfd3c0d5eaa60ca46181 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 12 Dec 2023 17:22:54 +0100 Subject: [PATCH 125/141] schemeRegex -> schemeNameRegex Scheme could be understood to include the typical `:` separator. --- src/libutil/url-parts.hh | 2 +- src/libutil/url.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh index 5c5a30dc2..07bc8d0cd 100644 --- a/src/libutil/url-parts.hh +++ b/src/libutil/url-parts.hh @@ -8,7 +8,7 @@ namespace nix { // URI stuff. const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])"; -const static std::string schemeRegex = "(?:[a-z][a-z0-9+.-]*)"; +const static std::string schemeNameRegex = "(?:[a-z][a-z0-9+.-]*)"; const static std::string ipv6AddressSegmentRegex = "[0-9a-fA-F:]+(?:%\\w+)?"; const static std::string ipv6AddressRegex = "(?:\\[" + ipv6AddressSegmentRegex + "\\]|" + ipv6AddressSegmentRegex + ")"; const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])"; diff --git a/src/libutil/url.cc b/src/libutil/url.cc index f2d5f1782..e9acd67d0 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -14,7 +14,7 @@ std::regex revRegex(revRegexS, std::regex::ECMAScript); ParsedURL parseURL(const std::string & url) { static std::regex uriRegex( - "((" + schemeRegex + "):" + "((" + schemeNameRegex + "):" + "(?:(?://(" + authorityRegex + ")(" + absPathRegex + "))|(/?" + pathRegex + ")))" + "(?:\\?(" + queryRegex + "))?" + "(?:#(" + queryRegex + "))?", From 4eaeda6604e2f8977728f14415fe92350d047970 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 12 Dec 2023 17:43:54 +0100 Subject: [PATCH 126/141] isValidSchemeName: Use regex As requested by Eelco Dolstra. I think it used to be simpler. --- src/libutil/url.cc | 15 +++------------ tests/unit/libutil/url.cc | 5 +++++ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/libutil/url.cc b/src/libutil/url.cc index e9acd67d0..152c06d8e 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -3,7 +3,6 @@ #include "util.hh" #include "split.hh" #include "canon-path.hh" -#include "string.hh" namespace nix { @@ -187,17 +186,9 @@ std::string fixGitURL(const std::string & url) // https://www.rfc-editor.org/rfc/rfc3986#section-3.1 bool isValidSchemeName(std::string_view s) { - if (s.empty()) return false; - if (!isASCIIAlpha(s[0])) return false; - for (auto c : s.substr(1)) { - if (isASCIIAlpha(c)) continue; - if (isASCIIDigit(c)) continue; - if (c == '+') continue; - if (c == '-') continue; - if (c == '.') continue; - return false; - } - return true; + static std::regex regex(schemeNameRegex, std::regex::ECMAScript); + + return std::regex_match(s.begin(), s.end(), regex, std::regex_constants::match_default); } } diff --git a/tests/unit/libutil/url.cc b/tests/unit/libutil/url.cc index 09fa4e218..7d08f467e 100644 --- a/tests/unit/libutil/url.cc +++ b/tests/unit/libutil/url.cc @@ -360,6 +360,11 @@ TEST(nix, isValidSchemeName) { ASSERT_FALSE(isValidSchemeName(".file")); ASSERT_FALSE(isValidSchemeName("-file")); ASSERT_FALSE(isValidSchemeName("1file")); + // regex ok? + ASSERT_FALSE(isValidSchemeName("\nhttp")); + ASSERT_FALSE(isValidSchemeName("\nhttp\n")); + ASSERT_FALSE(isValidSchemeName("http\n")); + ASSERT_FALSE(isValidSchemeName("http ")); } } From 0b87ba50c08d83384e11a8e6db1e2f97fba4b61c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 12 Dec 2023 17:44:53 +0100 Subject: [PATCH 127/141] Revert "Add nix::isASCII*, locale-independent" This reverts commit 79eb2920bb51c7ec9528a403986e79f04738e2be. Not used at this time. --- src/libutil/string.hh | 17 ----------- tests/unit/libutil/string.cc | 59 ------------------------------------ 2 files changed, 76 deletions(-) delete mode 100644 src/libutil/string.hh delete mode 100644 tests/unit/libutil/string.cc diff --git a/src/libutil/string.hh b/src/libutil/string.hh deleted file mode 100644 index 16ef75643..000000000 --- a/src/libutil/string.hh +++ /dev/null @@ -1,17 +0,0 @@ -#pragma once - -namespace nix { - - /** Locale-independent version of std::islower(). */ - inline bool isASCIILower(char c) { return c >= 'a' && c <= 'z'; }; - - /** Locale-independent version of std::isupper(). */ - inline bool isASCIIUpper(char c) { return c >= 'A' && c <= 'Z'; }; - - /** Locale-independent version of std::isalpha(). */ - inline bool isASCIIAlpha(char c) { return isASCIILower(c) || isASCIIUpper(c); }; - - /** Locale-independent version of std::isdigit(). */ - inline bool isASCIIDigit(char c) { return c >= '0' && c <= '9'; }; - -} diff --git a/tests/unit/libutil/string.cc b/tests/unit/libutil/string.cc deleted file mode 100644 index 381f2cc15..000000000 --- a/tests/unit/libutil/string.cc +++ /dev/null @@ -1,59 +0,0 @@ -#include -#include "string.hh" - -namespace nix { - -TEST(string, isASCIILower) { - ASSERT_TRUE(isASCIILower('a')); - ASSERT_TRUE(isASCIILower('z')); - ASSERT_FALSE(isASCIILower('A')); - ASSERT_FALSE(isASCIILower('Z')); - ASSERT_FALSE(isASCIILower('0')); - ASSERT_FALSE(isASCIILower('9')); - ASSERT_FALSE(isASCIILower(' ')); - ASSERT_FALSE(isASCIILower('\n')); - ASSERT_FALSE(isASCIILower('\t')); - ASSERT_FALSE(isASCIILower(':')); -} - -TEST(string, isASCIIUpper) { - ASSERT_FALSE(isASCIIUpper('a')); - ASSERT_FALSE(isASCIIUpper('z')); - ASSERT_TRUE(isASCIIUpper('A')); - ASSERT_TRUE(isASCIIUpper('Z')); - ASSERT_FALSE(isASCIIUpper('0')); - ASSERT_FALSE(isASCIIUpper('9')); - ASSERT_FALSE(isASCIIUpper(' ')); - ASSERT_FALSE(isASCIIUpper('\n')); - ASSERT_FALSE(isASCIIUpper('\t')); - ASSERT_FALSE(isASCIIUpper(':')); -} - -TEST(string, isASCIIAlpha) { - ASSERT_TRUE(isASCIIAlpha('a')); - ASSERT_TRUE(isASCIIAlpha('z')); - ASSERT_TRUE(isASCIIAlpha('A')); - ASSERT_TRUE(isASCIIAlpha('Z')); - ASSERT_FALSE(isASCIIAlpha('0')); - ASSERT_FALSE(isASCIIAlpha('9')); - ASSERT_FALSE(isASCIIAlpha(' ')); - ASSERT_FALSE(isASCIIAlpha('\n')); - ASSERT_FALSE(isASCIIAlpha('\t')); - ASSERT_FALSE(isASCIIAlpha(':')); -} - -TEST(string, isASCIIDigit) { - ASSERT_FALSE(isASCIIDigit('a')); - ASSERT_FALSE(isASCIIDigit('z')); - ASSERT_FALSE(isASCIIDigit('A')); - ASSERT_FALSE(isASCIIDigit('Z')); - ASSERT_TRUE(isASCIIDigit('0')); - ASSERT_TRUE(isASCIIDigit('1')); - ASSERT_TRUE(isASCIIDigit('9')); - ASSERT_FALSE(isASCIIDigit(' ')); - ASSERT_FALSE(isASCIIDigit('\n')); - ASSERT_FALSE(isASCIIDigit('\t')); - ASSERT_FALSE(isASCIIDigit(':')); -} - -} \ No newline at end of file From 04f454f2a0e1bfb4fc0368872f215cb690df11bc Mon Sep 17 00:00:00 2001 From: SharzyL Date: Wed, 13 Dec 2023 10:30:28 +0800 Subject: [PATCH 128/141] fix: nix copy ssh-ng:// not respecting --substitute-on-destination --- src/libstore/remote-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index cc26c2a94..dd6347468 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -225,7 +225,7 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute conn->to << WorkerProto::Op::QueryValidPaths; WorkerProto::write(*this, *conn, paths); if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 27) { - conn->to << (settings.buildersUseSubstitutes ? 1 : 0); + conn->to << maybeSubstitute; } conn.processStderr(); return WorkerProto::Serialise::read(*this, *conn); From cc3913e4584beb19e7af00572db119d2638333d5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 Dec 2023 13:27:23 +0100 Subject: [PATCH 129/141] Remove unused variable --- src/libexpr/eval.hh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index f3f6d35b9..f452dcb9f 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -335,11 +335,6 @@ private: std::map> searchPathResolved; - /** - * Cache used by checkSourcePath(). - */ - std::unordered_map resolvedPaths; - /** * Cache used by prim_match(). */ From 103ca0bde5d4f32745d4c3aee534cf4aa0a69a9d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 Dec 2023 13:27:29 +0100 Subject: [PATCH 130/141] Improve SourcePath display --- src/libfetchers/input-accessor.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh index d5ac238b1..f385e6231 100644 --- a/src/libfetchers/input-accessor.hh +++ b/src/libfetchers/input-accessor.hh @@ -130,7 +130,7 @@ struct SourcePath { return accessor->getPhysicalPath(path); } std::string to_string() const - { return path.abs(); } + { return accessor->showPath(path); } /** * Append a `CanonPath` to this path. From faa4cae9aed4e9f8c40ed8c6fe00bd0216c3b0ea Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 Dec 2023 13:27:39 +0100 Subject: [PATCH 131/141] LibExprTest: Ignore $NIX_PATH Otherwise a broken $NIX_PATH can cause the test suite to fail. --- tests/unit/libexpr-support/tests/libexpr.hh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/libexpr-support/tests/libexpr.hh b/tests/unit/libexpr-support/tests/libexpr.hh index 968431446..d720cedde 100644 --- a/tests/unit/libexpr-support/tests/libexpr.hh +++ b/tests/unit/libexpr-support/tests/libexpr.hh @@ -8,6 +8,7 @@ #include "nixexpr.hh" #include "eval.hh" #include "eval-inline.hh" +#include "eval-settings.hh" #include "store-api.hh" #include "tests/libstore.hh" @@ -18,6 +19,7 @@ namespace nix { static void SetUpTestSuite() { LibStoreTest::SetUpTestSuite(); initGC(); + evalSettings.nixPath = {}; } protected: From 19ec1c9fd4d4bf6e941b046b8549ba2a1a690937 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 13 Dec 2023 15:15:30 +0100 Subject: [PATCH 132/141] Improve the unsafeGetAttrPos test We can use corepkgsFS->addFile() now to create a "real" position. --- tests/unit/libexpr/primops.cc | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/tests/unit/libexpr/primops.cc b/tests/unit/libexpr/primops.cc index 7485fa0d0..384d9924b 100644 --- a/tests/unit/libexpr/primops.cc +++ b/tests/unit/libexpr/primops.cc @@ -1,6 +1,8 @@ #include #include +#include "memory-input-accessor.hh" + #include "tests/libexpr.hh" namespace nix { @@ -148,10 +150,25 @@ namespace nix { } TEST_F(PrimOpTest, unsafeGetAttrPos) { - // The `y` attribute is at position - const char* expr = "builtins.unsafeGetAttrPos \"y\" { y = \"x\"; }"; + state.corepkgsFS->addFile(CanonPath("foo.nix"), "{ y = \"x\"; }"); + + auto expr = "builtins.unsafeGetAttrPos \"y\" (import )"; auto v = eval(expr); - ASSERT_THAT(v, IsNull()); + ASSERT_THAT(v, IsAttrsOfSize(3)); + + auto file = v.attrs->find(createSymbol("file")); + ASSERT_NE(file, nullptr); + ASSERT_THAT(*file->value, IsString()); + auto s = baseNameOf(file->value->string_view()); + ASSERT_EQ(s, "foo.nix"); + + auto line = v.attrs->find(createSymbol("line")); + ASSERT_NE(line, nullptr); + ASSERT_THAT(*line->value, IsIntEq(1)); + + auto column = v.attrs->find(createSymbol("column")); + ASSERT_NE(column, nullptr); + ASSERT_THAT(*column->value, IsIntEq(3)); } TEST_F(PrimOpTest, hasAttr) { From e76df8781417dad9ab4f0a6c3b28917e35f204bf Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 13 Dec 2023 11:26:14 -0500 Subject: [PATCH 133/141] Test `nix copy --substitute-on-destination` It works with both `ssh://` and `ssh-ng://` now since #9600 (and `ssh-ng:// didn't work before that). Also, by making the two tests share code, we nudge ourselves towards making sure there is feature parity. --- tests/functional/nix-copy-ssh-common.sh | 70 +++++++++++++++++++++++++ tests/functional/nix-copy-ssh-ng.sh | 16 +++--- tests/functional/nix-copy-ssh.sh | 19 +------ 3 files changed, 77 insertions(+), 28 deletions(-) create mode 100644 tests/functional/nix-copy-ssh-common.sh diff --git a/tests/functional/nix-copy-ssh-common.sh b/tests/functional/nix-copy-ssh-common.sh new file mode 100644 index 000000000..cc8314ff7 --- /dev/null +++ b/tests/functional/nix-copy-ssh-common.sh @@ -0,0 +1,70 @@ +proto=$1 +shift +(( $# == 0 )) + +clearStore +clearCache + +mkdir -p $TEST_ROOT/stores + +# Create path to copy back and forth +outPath=$(nix-build --no-out-link dependencies.nix) + +storeQueryParam="store=${NIX_STORE_DIR}" + +realQueryParam () { + echo "real=$1$NIX_STORE_DIR" +} + +remoteRoot="$TEST_ROOT/stores/$proto" + +clearRemoteStore () { + chmod -R u+w "$remoteRoot" || true + rm -rf "$remoteRoot" +} + +clearRemoteStore + +remoteStore="${proto}://localhost?${storeQueryParam}&remote-store=${remoteRoot}%3f${storeQueryParam}%26$(realQueryParam "$remoteRoot")" + +# Copy to store + +args=() +if [[ "$proto" == "ssh-ng" ]]; then + # TODO investigate discrepancy + args+=(--no-check-sigs) +fi + +[ ! -f ${remoteRoot}${outPath}/foobar ] +nix copy "${args[@]}" --to "$remoteStore" $outPath +[ -f ${remoteRoot}${outPath}/foobar ] + +# Copy back from store + +clearStore + +[ ! -f $outPath/foobar ] +nix copy --no-check-sigs --from "$remoteStore" $outPath +[ -f $outPath/foobar ] + +# Check --substitute-on-destination, avoid corrupted store + +clearRemoteStore + +corruptedRoot=$TEST_ROOT/stores/corrupted +corruptedStore="${corruptedRoot}?${storeQueryParam}&$(realQueryParam "$corruptedRoot")" + +# Copy it to the corrupted store +nix copy --no-check-sigs "$outPath" --to "$corruptedStore" + +# Corrupt it in there +corruptPath="${corruptedRoot}${outPath}" +chmod +w "$corruptPath" +echo "not supposed to be here" > "$corruptPath/foobarbaz" +chmod -w "$corruptPath" + +# Copy from the corrupted store with the regular store as a +# substituter. It must use the substituter not the source store in +# order to avoid errors. +NIX_CONFIG=$(echo -e "substituters = local\nrequire-sigs = false") \ + nix copy --no-check-sigs --from "$corruptedStore" --to "$remoteStore" --substitute-on-destination "$outPath" diff --git a/tests/functional/nix-copy-ssh-ng.sh b/tests/functional/nix-copy-ssh-ng.sh index 463b5e0c4..62e99cd24 100644 --- a/tests/functional/nix-copy-ssh-ng.sh +++ b/tests/functional/nix-copy-ssh-ng.sh @@ -1,18 +1,14 @@ source common.sh -clearStore -clearCache +source nix-copy-ssh-common.sh "ssh-ng" -remoteRoot=$TEST_ROOT/store2 -chmod -R u+w "$remoteRoot" || true -rm -rf "$remoteRoot" +clearStore +clearRemoteStore outPath=$(nix-build --no-out-link dependencies.nix) -nix store info --store "ssh-ng://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" +nix store info --store "$remoteStore" # Regression test for https://github.com/NixOS/nix/issues/6253 -nix copy --to "ssh-ng://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath --no-check-sigs & -nix copy --to "ssh-ng://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath --no-check-sigs - -[ -f $remoteRoot$outPath/foobar ] +nix copy --to "$remoteStore" $outPath --no-check-sigs & +nix copy --to "$remoteStore" $outPath --no-check-sigs diff --git a/tests/functional/nix-copy-ssh.sh b/tests/functional/nix-copy-ssh.sh index eb801548d..12e8346bc 100644 --- a/tests/functional/nix-copy-ssh.sh +++ b/tests/functional/nix-copy-ssh.sh @@ -1,20 +1,3 @@ source common.sh -clearStore -clearCache - -remoteRoot=$TEST_ROOT/store2 -chmod -R u+w "$remoteRoot" || true -rm -rf "$remoteRoot" - -outPath=$(nix-build --no-out-link dependencies.nix) - -nix copy --to "ssh://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath - -[ -f $remoteRoot$outPath/foobar ] - -clearStore - -nix copy --no-check-sigs --from "ssh://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath - -[ -f $outPath/foobar ] +source nix-copy-ssh-common.sh "ssh" From 19573f1b05b7d3ccfd07c9c351396494d488ab2d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 13 Dec 2023 15:33:15 -0500 Subject: [PATCH 134/141] Restore comment --- scripts/binary-tarball.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/binary-tarball.nix b/scripts/binary-tarball.nix index 32e811c94..104189b0c 100644 --- a/scripts/binary-tarball.nix +++ b/scripts/binary-tarball.nix @@ -14,6 +14,7 @@ let inherit (nix) version; env = { + #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck; meta.description = "Distribution-independent Nix bootstrap binaries for ${system}"; }; From f10f0f1b50228e09ad587a7c550df586061e4514 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 13 Dec 2023 20:41:20 +0000 Subject: [PATCH 135/141] Move `lowdown.nix` to `misc/` --- flake.nix | 2 +- lowdown.nix => misc/lowdown.nix | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename lowdown.nix => misc/lowdown.nix (100%) diff --git a/flake.nix b/flake.nix index 0cdd2b41f..c7ff7eb64 100644 --- a/flake.nix +++ b/flake.nix @@ -167,7 +167,7 @@ ''; }; - lowdown-nix = final.callPackage ./lowdown.nix { + lowdown-nix = final.callPackage ./misc/lowdown.nix { inherit lowdown-src stdenv; }; diff --git a/lowdown.nix b/misc/lowdown.nix similarity index 100% rename from lowdown.nix rename to misc/lowdown.nix From bf5804d46a0d0aa5eb40107b6eaeec4e95bbd4a2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 13 Dec 2023 20:41:41 +0000 Subject: [PATCH 136/141] flake.nix: Delete uneeded `attrs0` binding --- package.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.nix b/package.nix index 7c1ba3130..8fc4be328 100644 --- a/package.nix +++ b/package.nix @@ -92,7 +92,7 @@ , __forDefaults ? { canRunInstalled = doBuild && stdenv.buildPlatform.canExecute stdenv.hostPlatform; } -} @ attrs0: +}: let version = lib.fileContents ./.version + versionSuffix; From 28f2f3136d19ef7de4c6acd9678aef72e80d4fb8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 13 Dec 2023 20:47:36 +0000 Subject: [PATCH 137/141] Delete stray `install_name_tool` call --- package.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/package.nix b/package.nix index 8fc4be328..0b5b512c7 100644 --- a/package.nix +++ b/package.nix @@ -320,7 +320,6 @@ in { -change ${boost}/lib/libboost_context.dylib \ $out/lib/libboost_context.dylib \ $out/lib/libnixutil.dylib - install_name_tool '' ) + lib.optionalString enableInternalAPIDocs '' mkdir -p ''${!outputDoc}/nix-support From 2d24875fe4aa7f31d15acfc29b9aa5c45109f99d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 13 Dec 2023 20:49:31 +0000 Subject: [PATCH 138/141] package.nix: Avoid `${..}` for conditional strings Using `+` is Nixpkgs standard ideom for this, and helps avoid needless rebuilds somewhat. --- package.nix | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/package.nix b/package.nix index 0b5b512c7..0b2ff43b0 100644 --- a/package.nix +++ b/package.nix @@ -254,25 +254,25 @@ in { disallowedReferences = [ boost ]; - preConfigure = lib.optionalString (doBuild && ! stdenv.hostPlatform.isStatic) '' - # Copy libboost_context so we don't get all of Boost in our closure. - # https://github.com/NixOS/nixpkgs/issues/45462 - mkdir -p $out/lib - cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*,libboost_regex*} $out/lib - rm -f $out/lib/*.a - ${lib.optionalString stdenv.hostPlatform.isLinux '' + preConfigure = lib.optionalString (doBuild && ! stdenv.hostPlatform.isStatic) ( + '' + # Copy libboost_context so we don't get all of Boost in our closure. + # https://github.com/NixOS/nixpkgs/issues/45462 + mkdir -p $out/lib + cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*,libboost_regex*} $out/lib + rm -f $out/lib/*.a + '' + lib.optionalString stdenv.hostPlatform.isLinux '' chmod u+w $out/lib/*.so.* patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* - ''} - ${lib.optionalString stdenv.hostPlatform.isDarwin '' + '' + lib.optionalString stdenv.hostPlatform.isDarwin '' for LIB in $out/lib/*.dylib; do chmod u+w $LIB install_name_tool -id $LIB $LIB install_name_tool -delete_rpath ${boost}/lib/ $LIB || true done install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib - ''} - ''; + '' + ); configureFlags = [ "--sysconfdir=/etc" From 7b29b44d8e62f686aa9fbfafe53be959cdba03cb Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 13 Dec 2023 16:22:35 -0500 Subject: [PATCH 139/141] Remove custom lowdown This was last upgraded in 788008385ef5bf7edb799977525d6f73f02c76bc, but the version in Nixpkgs is a now a lot newer. I think the custom was added to get ahead of Nixpkgs before, and so now that we are in fact behind, it is no longer needed. --- flake.lock | 17 ----------------- flake.nix | 11 +---------- misc/lowdown.nix | 22 ---------------------- 3 files changed, 1 insertion(+), 49 deletions(-) delete mode 100644 misc/lowdown.nix diff --git a/flake.lock b/flake.lock index 3cb9e72c9..db1a72c14 100644 --- a/flake.lock +++ b/flake.lock @@ -32,22 +32,6 @@ "type": "github" } }, - "lowdown-src": { - "flake": false, - "locked": { - "lastModified": 1633514407, - "narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=", - "owner": "kristapsdz", - "repo": "lowdown", - "rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8", - "type": "github" - }, - "original": { - "owner": "kristapsdz", - "repo": "lowdown", - "type": "github" - } - }, "nixpkgs": { "locked": { "lastModified": 1701355166, @@ -84,7 +68,6 @@ "inputs": { "flake-compat": "flake-compat", "libgit2": "libgit2", - "lowdown-src": "lowdown-src", "nixpkgs": "nixpkgs", "nixpkgs-regression": "nixpkgs-regression" } diff --git a/flake.nix b/flake.nix index c7ff7eb64..eb3846564 100644 --- a/flake.nix +++ b/flake.nix @@ -13,11 +13,10 @@ # to remove the `nix.checkAllErrors = false;` line in the tests. inputs.nixpkgs.url = "github:NixOS/nixpkgs/staging-23.05"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; - inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; }; - outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src, libgit2, ... }: + outputs = { self, nixpkgs, nixpkgs-regression, libgit2, ... }: let inherit (nixpkgs) lib; @@ -140,9 +139,6 @@ { nixStable = prev.nix; - # Forward from the previous stage as we don’t want it to pick the lowdown override - inherit (prev) nixUnstable; - default-busybox-sandbox-shell = final.busybox.override { useMusl = true; enableStatic = true; @@ -167,10 +163,6 @@ ''; }; - lowdown-nix = final.callPackage ./misc/lowdown.nix { - inherit lowdown-src stdenv; - }; - libgit2-nix = final.libgit2.overrideAttrs (attrs: { src = libgit2; version = libgit2.lastModifiedDate; @@ -208,7 +200,6 @@ officialRelease = false; boehmgc = final.boehmgc-nix; libgit2 = final.libgit2-nix; - lowdown = final.lowdown-nix; busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell; changelog-d = final.changelog-d-nix; } // { diff --git a/misc/lowdown.nix b/misc/lowdown.nix deleted file mode 100644 index 5f469fad5..000000000 --- a/misc/lowdown.nix +++ /dev/null @@ -1,22 +0,0 @@ -{ lib -, stdenv -, which -, lowdown-src -}: - -stdenv.mkDerivation rec { - name = "lowdown-0.9.0"; - - src = lowdown-src; - - outputs = [ "out" "bin" "dev" ]; - - nativeBuildInputs = [ which ]; - - configurePhase = '' - ${lib.optionalString (stdenv.isDarwin && stdenv.isAarch64) "echo \"HAVE_SANDBOX_INIT=false\" > configure.local"} - ./configure \ - PREFIX=${placeholder "dev"} \ - BINDIR=${placeholder "bin"}/bin - ''; -} From 1e3d8118401d80da54fb64641e606042c3499e4d Mon Sep 17 00:00:00 2001 From: Ramses Date: Wed, 13 Dec 2023 22:37:17 +0100 Subject: [PATCH 140/141] worker protocol: serialise cgroup stats in `BuildResult` (#9598) By doing so, they get reported when building through the daemon via either `unix://` or `ssh-ng://`. --- doc/manual/rl-next/cgroup-stats.md | 8 +++ src/libstore/worker-protocol.cc | 34 ++++++++++ src/libstore/worker-protocol.hh | 6 +- .../worker-protocol/build-result-1.37.bin | Bin 0 -> 808 bytes tests/unit/libstore/worker-protocol.cc | 61 ++++++++++++++++-- 5 files changed, 101 insertions(+), 8 deletions(-) create mode 100644 doc/manual/rl-next/cgroup-stats.md create mode 100644 tests/unit/libstore/data/worker-protocol/build-result-1.37.bin diff --git a/doc/manual/rl-next/cgroup-stats.md b/doc/manual/rl-next/cgroup-stats.md new file mode 100644 index 000000000..00853a0f8 --- /dev/null +++ b/doc/manual/rl-next/cgroup-stats.md @@ -0,0 +1,8 @@ +--- +synopsis: Include cgroup stats when building through the daemon +prs: 9598 +--- + +Nix now also reports cgroup statistics when building through the nix daemon and when doing remote builds using ssh-ng, +if both sides of the connection are this version of Nix or newer. + diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index 2a379e75e..a50259d24 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -7,6 +7,7 @@ #include "archive.hh" #include "path-info.hh" +#include #include namespace nix { @@ -47,6 +48,31 @@ void WorkerProto::Serialise>::write(const StoreDirCon } +std::optional WorkerProto::Serialise>::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) +{ + auto tag = readNum(conn.from); + switch (tag) { + case 0: + return std::nullopt; + case 1: + return std::optional{std::chrono::microseconds(readNum(conn.from))}; + default: + throw Error("Invalid optional tag from remote"); + } +} + +void WorkerProto::Serialise>::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const std::optional & optDuration) +{ + if (!optDuration.has_value()) { + conn.to << uint8_t{0}; + } else { + conn.to + << uint8_t{1} + << optDuration.value().count(); + } +} + + DerivedPath WorkerProto::Serialise::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) { auto s = readString(conn.from); @@ -110,6 +136,10 @@ BuildResult WorkerProto::Serialise::read(const StoreDirConfig & sto >> res.startTime >> res.stopTime; } + if (GET_PROTOCOL_MINOR(conn.version) >= 37) { + res.cpuUser = WorkerProto::Serialise>::read(store, conn); + res.cpuSystem = WorkerProto::Serialise>::read(store, conn); + } if (GET_PROTOCOL_MINOR(conn.version) >= 28) { auto builtOutputs = WorkerProto::Serialise::read(store, conn); for (auto && [output, realisation] : builtOutputs) @@ -132,6 +162,10 @@ void WorkerProto::Serialise::write(const StoreDirConfig & store, Wo << res.startTime << res.stopTime; } + if (GET_PROTOCOL_MINOR(conn.version) >= 37) { + WorkerProto::write(store, conn, res.cpuUser); + WorkerProto::write(store, conn, res.cpuSystem); + } if (GET_PROTOCOL_MINOR(conn.version) >= 28) { DrvOutputs builtOutputs; for (auto & [output, realisation] : res.builtOutputs) diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index c26914289..91d277b77 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -1,6 +1,8 @@ #pragma once ///@file +#include + #include "common-protocol.hh" namespace nix { @@ -9,7 +11,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION (1 << 8 | 36) +#define PROTOCOL_VERSION (1 << 8 | 37) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -214,6 +216,8 @@ template<> DECLARE_WORKER_SERIALISER(UnkeyedValidPathInfo); template<> DECLARE_WORKER_SERIALISER(std::optional); +template<> +DECLARE_WORKER_SERIALISER(std::optional); template DECLARE_WORKER_SERIALISER(std::vector); diff --git a/tests/unit/libstore/data/worker-protocol/build-result-1.37.bin b/tests/unit/libstore/data/worker-protocol/build-result-1.37.bin new file mode 100644 index 0000000000000000000000000000000000000000..7d6e43fff2f593c9669897f1594ef2526293b866 GIT binary patch literal 808 zcmc(b!A=4(5QbNYCw&D7HXbqAZP|9&SMZ?mYCOy`Q??0dfh{3~@J_yn?_}3497ssi zkk~`NPG{zy$$yh{=Qh&1p+SP-rryS%zu_*nozv~b{8i*2l1Kg&hyFwTsm?J^pZ&Jx z7(XWuZG7Ec;XHLnni_a6OQ{Pv(Gvn*a8V&-GN)9gN@`4z#zM+q67p$E1#+ya@Ky@P zI?Ja*37y|pu=-Z~h`Kw5v>=OQ{VT!TG~kW14J&v15i`h2cEQPP#N67yfUkq@EZeAh nE0*W@7*-7pjhR{S>lKBa-ro0@_Cq`OPkw~Szw@JOIPv2Pj)JK^ literal 0 HcmV?d00001 diff --git a/tests/unit/libstore/worker-protocol.cc b/tests/unit/libstore/worker-protocol.cc index 91f804f0c..2b2e559a9 100644 --- a/tests/unit/libstore/worker-protocol.cc +++ b/tests/unit/libstore/worker-protocol.cc @@ -280,13 +280,60 @@ VERSIONED_CHARACTERIZATION_TEST( }, .startTime = 30, .stopTime = 50, -#if 0 - // These fields are not yet serialized. - // FIXME Include in next version of protocol or document - // why they are skipped. - .cpuUser = std::chrono::milliseconds(500s), - .cpuSystem = std::chrono::milliseconds(604s), -#endif + }, + }; + t; + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + buildResult_1_37, + "build-result-1.37", + 1 << 8 | 37, + ({ + using namespace std::literals::chrono_literals; + std::tuple t { + BuildResult { + .status = BuildResult::OutputRejected, + .errorMsg = "no idea why", + }, + BuildResult { + .status = BuildResult::NotDeterministic, + .errorMsg = "no idea why", + .timesBuilt = 3, + .isNonDeterministic = true, + .startTime = 30, + .stopTime = 50, + }, + BuildResult { + .status = BuildResult::Built, + .timesBuilt = 1, + .builtOutputs = { + { + "foo", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "foo", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + }, + { + "bar", + { + .id = DrvOutput { + .drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "bar", + }, + .outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar" }, + }, + }, + }, + .startTime = 30, + .stopTime = 50, + .cpuUser = std::chrono::microseconds(500s), + .cpuSystem = std::chrono::microseconds(604s), }, }; t; From 6ed803737c587a0cc9026093c941c1d1172fa5dc Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Wed, 13 Dec 2023 14:02:52 -0800 Subject: [PATCH 141/141] Use `--with-boost` on macOS `configureFlags` only included `--with-boost` on Linux, which makes local builds as outlined in `doc/manual/src/contributing/hacking.md` fail when performed on macOS. --- package.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.nix b/package.nix index 0b2ff43b0..24395b484 100644 --- a/package.nix +++ b/package.nix @@ -284,8 +284,9 @@ in { ] ++ lib.optionals installUnitTests [ "--with-check-bin-dir=${builtins.placeholder "check"}/bin" "--with-check-lib-dir=${builtins.placeholder "check"}/lib" - ] ++ lib.optionals (doBuild && stdenv.isLinux) [ + ] ++ lib.optionals (doBuild) [ "--with-boost=${boost}/lib" + ] ++ lib.optionals (doBuild && stdenv.isLinux) [ "--with-sandbox-shell=${busybox-sandbox-shell}/bin/busybox" ] ++ lib.optional (doBuild && stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) "LDFLAGS=-fuse-ld=gold"