From 0e18254aa81b9315c13d6ae736cb38666d19f122 Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Wed, 5 Apr 2023 17:05:17 +0800 Subject: [PATCH 01/37] Fix shutdown behavior and resource management for recursive-nix on macOS Previously, we relied on the `shutdown()` function to terminate `accept()` calls on a listening socket. However, this approach did not work on macOS as the waiting `accept()` call is not considered a connected socket, resulting in an `ENOTCONN` error. Instead, we now close the listening socket to terminate the `accept()` call. Additionally, we fixed a resource management issue where we set the `daemonSocket` variable to -1, triggering resource cleanup and causing the `stopDaemon` function to be called twice. This resulted in errors as the socket was already closed by the time the second `stopDaemon` call was made. Instead of setting `daemonSocket` to -1, we now release the socket using the `release()` method on a unique pointer. This properly transfers ownership and allows for correct resource cleanup. These changes ensure proper behavior and resource management for the recursive-nix feature on macOS. --- src/libstore/build/local-derivation-goal.cc | 23 +++++++++++++++++---- tests/recursive.sh | 3 --- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 21cd6e7ee..4b978f2a4 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1457,7 +1457,7 @@ void LocalDerivationGoal::startDaemon() (struct sockaddr *) &remoteAddr, &remoteAddrLen); if (!remote) { if (errno == EINTR || errno == EAGAIN) continue; - if (errno == EINVAL) break; + if (errno == EINVAL || errno == ECONNABORTED) break; throw SysError("accepting connection"); } @@ -1487,8 +1487,22 @@ void LocalDerivationGoal::startDaemon() void LocalDerivationGoal::stopDaemon() { - if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1) - throw SysError("shutting down daemon socket"); + if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1) { + // According to the POSIX standard, the 'shutdown' function should + // return an ENOTCONN error when attempting to shut down a socket that + // hasn't been connected yet. This situation occurs when the 'accept' + // function is called on a socket without any accepted connections, + // leaving the socket unconnected. While Linux doesn't seem to produce + // an error for sockets that have only been accepted, more + // POSIX-compliant operating systems like OpenBSD, macOS, and others do + // return the ENOTCONN error. Therefore, we handle this error here to + // avoid raising an exception for compliant behaviour. + if (errno == ENOTCONN) { + daemonSocket.close(); + } else { + throw SysError("shutting down daemon socket"); + } + } if (daemonThread.joinable()) daemonThread.join(); @@ -1499,7 +1513,8 @@ void LocalDerivationGoal::stopDaemon() thread.join(); daemonWorkerThreads.clear(); - daemonSocket = -1; + // release the socket. + daemonSocket.close(); } diff --git a/tests/recursive.sh b/tests/recursive.sh index 638f06f85..b661422ed 100644 --- a/tests/recursive.sh +++ b/tests/recursive.sh @@ -3,9 +3,6 @@ source common.sh sed -i 's/experimental-features .*/& recursive-nix/' "$NIX_CONF_DIR"/nix.conf restartDaemon -# FIXME -if [[ $(uname) != Linux ]]; then skipTest "Not running Linux"; fi - clearStore rm -f $TEST_ROOT/result From 3ebe1341abe1b0ad59bd4925517af18d9200f818 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 17 Mar 2023 15:51:08 +0100 Subject: [PATCH 02/37] Make `RewritingSink` accept a map of rewrites Giving it the same semantics as `rewriteStrings`. Also add some tests for it --- src/libexpr/primops.cc | 2 +- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/path-references.cc | 73 +++++++++++++++++ src/libstore/path-references.hh | 25 ++++++ src/{libstore => libutil}/references.cc | 86 +++++---------------- src/{libstore => libutil}/references.hh | 23 +----- src/libutil/tests/references.cc | 46 +++++++++++ 7 files changed, 169 insertions(+), 88 deletions(-) create mode 100644 src/libstore/path-references.cc create mode 100644 src/libstore/path-references.hh rename src/{libstore => libutil}/references.cc (61%) rename src/{libstore => libutil}/references.hh (61%) create mode 100644 src/libutil/tests/references.cc diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index cfae1e5f8..572e76ec6 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -6,7 +6,7 @@ #include "globals.hh" #include "json-to-value.hh" #include "names.hh" -#include "references.hh" +#include "path-references.hh" #include "store-api.hh" #include "util.hh" #include "value-to-json.hh" diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 05d6685da..6f7ab8a3d 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -4,7 +4,7 @@ #include "worker.hh" #include "builtins.hh" #include "builtins/buildenv.hh" -#include "references.hh" +#include "path-references.hh" #include "finally.hh" #include "util.hh" #include "archive.hh" diff --git a/src/libstore/path-references.cc b/src/libstore/path-references.cc new file mode 100644 index 000000000..33cf66ce3 --- /dev/null +++ b/src/libstore/path-references.cc @@ -0,0 +1,73 @@ +#include "path-references.hh" +#include "hash.hh" +#include "util.hh" +#include "archive.hh" + +#include +#include +#include +#include + + +namespace nix { + + +PathRefScanSink::PathRefScanSink(StringSet && hashes, std::map && backMap) + : RefScanSink(std::move(hashes)) + , backMap(std::move(backMap)) +{ } + +PathRefScanSink PathRefScanSink::fromPaths(const StorePathSet & refs) +{ + StringSet hashes; + std::map backMap; + + for (auto & i : refs) { + std::string hashPart(i.hashPart()); + auto inserted = backMap.emplace(hashPart, i).second; + assert(inserted); + hashes.insert(hashPart); + } + + return PathRefScanSink(std::move(hashes), std::move(backMap)); +} + +StorePathSet PathRefScanSink::getResultPaths() +{ + /* Map the hashes found back to their store paths. */ + StorePathSet found; + for (auto & i : getResult()) { + auto j = backMap.find(i); + assert(j != backMap.end()); + found.insert(j->second); + } + + return found; +} + + +std::pair scanForReferences( + const std::string & path, + const StorePathSet & refs) +{ + HashSink hashSink { htSHA256 }; + auto found = scanForReferences(hashSink, path, refs); + auto hash = hashSink.finish(); + return std::pair(found, hash); +} + +StorePathSet scanForReferences( + Sink & toTee, + const Path & path, + const StorePathSet & refs) +{ + PathRefScanSink refsSink = PathRefScanSink::fromPaths(refs); + TeeSink sink { refsSink, toTee }; + + /* Look for the hashes in the NAR dump of the path. */ + dumpPath(path, sink); + + return refsSink.getResultPaths(); +} + +} diff --git a/src/libstore/path-references.hh b/src/libstore/path-references.hh new file mode 100644 index 000000000..7b44e3261 --- /dev/null +++ b/src/libstore/path-references.hh @@ -0,0 +1,25 @@ +#pragma once + +#include "references.hh" +#include "path.hh" + +namespace nix { + +std::pair scanForReferences(const Path & path, const StorePathSet & refs); + +StorePathSet scanForReferences(Sink & toTee, const Path & path, const StorePathSet & refs); + +class PathRefScanSink : public RefScanSink +{ + std::map backMap; + + PathRefScanSink(StringSet && hashes, std::map && backMap); + +public: + + static PathRefScanSink fromPaths(const StorePathSet & refs); + + StorePathSet getResultPaths(); +}; + +} diff --git a/src/libstore/references.cc b/src/libutil/references.cc similarity index 61% rename from src/libstore/references.cc rename to src/libutil/references.cc index 345f4528b..74003584a 100644 --- a/src/libstore/references.cc +++ b/src/libutil/references.cc @@ -6,6 +6,7 @@ #include #include #include +#include namespace nix { @@ -66,69 +67,20 @@ void RefScanSink::operator () (std::string_view data) } -PathRefScanSink::PathRefScanSink(StringSet && hashes, std::map && backMap) - : RefScanSink(std::move(hashes)) - , backMap(std::move(backMap)) -{ } - -PathRefScanSink PathRefScanSink::fromPaths(const StorePathSet & refs) -{ - StringSet hashes; - std::map backMap; - - for (auto & i : refs) { - std::string hashPart(i.hashPart()); - auto inserted = backMap.emplace(hashPart, i).second; - assert(inserted); - hashes.insert(hashPart); - } - - return PathRefScanSink(std::move(hashes), std::move(backMap)); -} - -StorePathSet PathRefScanSink::getResultPaths() -{ - /* Map the hashes found back to their store paths. */ - StorePathSet found; - for (auto & i : getResult()) { - auto j = backMap.find(i); - assert(j != backMap.end()); - found.insert(j->second); - } - - return found; -} - - -std::pair scanForReferences( - const std::string & path, - const StorePathSet & refs) -{ - HashSink hashSink { htSHA256 }; - auto found = scanForReferences(hashSink, path, refs); - auto hash = hashSink.finish(); - return std::pair(found, hash); -} - -StorePathSet scanForReferences( - Sink & toTee, - const Path & path, - const StorePathSet & refs) -{ - PathRefScanSink refsSink = PathRefScanSink::fromPaths(refs); - TeeSink sink { refsSink, toTee }; - - /* Look for the hashes in the NAR dump of the path. */ - dumpPath(path, sink); - - return refsSink.getResultPaths(); -} - - RewritingSink::RewritingSink(const std::string & from, const std::string & to, Sink & nextSink) - : from(from), to(to), nextSink(nextSink) + : RewritingSink({{from, to}}, nextSink) { - assert(from.size() == to.size()); +} + +RewritingSink::RewritingSink(const StringMap & rewrites, Sink & nextSink) + : rewrites(rewrites), nextSink(nextSink) +{ + long unsigned int maxRewriteSize = 0; + for (auto & [from, to] : rewrites) { + assert(from.size() == to.size()); + maxRewriteSize = std::max(maxRewriteSize, from.size()); + } + this->maxRewriteSize = maxRewriteSize; } void RewritingSink::operator () (std::string_view data) @@ -136,13 +88,13 @@ void RewritingSink::operator () (std::string_view data) std::string s(prev); s.append(data); - size_t j = 0; - while ((j = s.find(from, j)) != std::string::npos) { - matches.push_back(pos + j); - s.replace(j, from.size(), to); - } + s = rewriteStrings(s, rewrites); - prev = s.size() < from.size() ? s : std::string(s, s.size() - from.size() + 1, from.size() - 1); + prev = s.size() < maxRewriteSize + ? s + : maxRewriteSize == 0 + ? "" + : std::string(s, s.size() - maxRewriteSize + 1, maxRewriteSize - 1); auto consumed = s.size() - prev.size(); diff --git a/src/libstore/references.hh b/src/libutil/references.hh similarity index 61% rename from src/libstore/references.hh rename to src/libutil/references.hh index 52d71b333..ffd730e7b 100644 --- a/src/libstore/references.hh +++ b/src/libutil/references.hh @@ -2,14 +2,9 @@ ///@file #include "hash.hh" -#include "path.hh" namespace nix { -std::pair scanForReferences(const Path & path, const StorePathSet & refs); - -StorePathSet scanForReferences(Sink & toTee, const Path & path, const StorePathSet & refs); - class RefScanSink : public Sink { StringSet hashes; @@ -28,28 +23,18 @@ public: void operator () (std::string_view data) override; }; -class PathRefScanSink : public RefScanSink -{ - std::map backMap; - - PathRefScanSink(StringSet && hashes, std::map && backMap); - -public: - - static PathRefScanSink fromPaths(const StorePathSet & refs); - - StorePathSet getResultPaths(); -}; - struct RewritingSink : Sink { - std::string from, to, prev; + const StringMap rewrites; + long unsigned int maxRewriteSize; + std::string prev; Sink & nextSink; uint64_t pos = 0; std::vector matches; RewritingSink(const std::string & from, const std::string & to, Sink & nextSink); + RewritingSink(const StringMap & rewrites, Sink & nextSink); void operator () (std::string_view data) override; diff --git a/src/libutil/tests/references.cc b/src/libutil/tests/references.cc new file mode 100644 index 000000000..a517d9aa1 --- /dev/null +++ b/src/libutil/tests/references.cc @@ -0,0 +1,46 @@ +#include "references.hh" +#include + +namespace nix { + +using std::string; + +struct RewriteParams { + string originalString, finalString; + StringMap rewrites; + + friend std::ostream& operator<<(std::ostream& os, const RewriteParams& bar) { + StringSet strRewrites; + for (auto & [from, to] : bar.rewrites) + strRewrites.insert(from + "->" + to); + return os << + "OriginalString: " << bar.originalString << std::endl << + "Rewrites: " << concatStringsSep(",", strRewrites) << std::endl << + "Expected result: " << bar.finalString; + } +}; + +class RewriteTest : public ::testing::TestWithParam { +}; + +TEST_P(RewriteTest, IdentityRewriteIsIdentity) { + RewriteParams param = GetParam(); + StringSink rewritten; + auto rewriter = RewritingSink(param.rewrites, rewritten); + rewriter(param.originalString); + rewriter.flush(); + ASSERT_EQ(rewritten.s, param.finalString); +} + +INSTANTIATE_TEST_CASE_P( + references, + RewriteTest, + ::testing::Values( + RewriteParams{ "foooo", "baroo", {{"foo", "bar"}, {"bar", "baz"}}}, + RewriteParams{ "foooo", "bazoo", {{"fou", "bar"}, {"foo", "baz"}}}, + RewriteParams{ "foooo", "foooo", {}} + ) +); + +} + From a917fb0d53544f7fd29f24d5984702296d457347 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 17 Mar 2023 15:51:08 +0100 Subject: [PATCH 03/37] Use a RewritingSink in derivation goal Possibly this will make it stream --- src/libstore/build/local-derivation-goal.cc | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 6f7ab8a3d..4650b8bae 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2384,13 +2384,16 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() if (!outputRewrites.empty()) { debug("rewriting hashes in '%1%'; cross fingers", actualPath); - /* FIXME: this is in-memory. */ - StringSink sink; - dumpPath(actualPath, sink); + /* FIXME: Is this actually streaming? */ + auto source = sinkToSource([&](Sink & nextSink) { + RewritingSink rsink(outputRewrites, nextSink); + dumpPath(actualPath, rsink); + rsink.flush(); + }); + Path tmpPath = actualPath + ".tmp"; + restorePath(tmpPath, *source); deletePath(actualPath); - sink.s = rewriteStrings(sink.s, outputRewrites); - StringSource source(sink.s); - restorePath(actualPath, source); + movePath(tmpPath, actualPath); /* FIXME: set proper permissions in restorePath() so we don't have to do another traversal. */ From 34e1b464f0054623334355aac94cb1a3e4c35a96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 17 Mar 2023 15:51:08 +0100 Subject: [PATCH 04/37] Normalize the hash-rewriting process when building derivations --- src/libstore/build/local-derivation-goal.cc | 28 ++++++++++----------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 4650b8bae..e9fe1eb73 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2379,14 +2379,14 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() continue; auto references = *referencesOpt; - auto rewriteOutput = [&]() { + auto rewriteOutput = [&](const StringMap & rewrites) { /* Apply hash rewriting if necessary. */ - if (!outputRewrites.empty()) { + if (!rewrites.empty()) { debug("rewriting hashes in '%1%'; cross fingers", actualPath); /* FIXME: Is this actually streaming? */ auto source = sinkToSource([&](Sink & nextSink) { - RewritingSink rsink(outputRewrites, nextSink); + RewritingSink rsink(rewrites, nextSink); dumpPath(actualPath, rsink); rsink.flush(); }); @@ -2442,7 +2442,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() "since recursive hashing is not enabled (one of outputHashMode={flat,text} is true)", actualPath); } - rewriteOutput(); + rewriteOutput(outputRewrites); /* FIXME optimize and deduplicate with addToStore */ std::string oldHashPart { scratchPath->hashPart() }; HashModuloSink caSink { outputHash.hashType, oldHashPart }; @@ -2480,16 +2480,14 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() Hash::dummy, }; if (*scratchPath != newInfo0.path) { - // Also rewrite the output path - auto source = sinkToSource([&](Sink & nextSink) { - RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink); - dumpPath(actualPath, rsink2); - rsink2.flush(); - }); - Path tmpPath = actualPath + ".tmp"; - restorePath(tmpPath, *source); - deletePath(actualPath); - movePath(tmpPath, actualPath); + // If the path has some self-references, we need to rewrite + // them. + // (note that this doesn't invalidate the ca hash we calculated + // above because it's computed *modulo the self-references*, so + // it already takes this rewrite into account). + rewriteOutput( + StringMap{{oldHashPart, + std::string(newInfo0.path.hashPart())}}); } HashResult narHashAndSize = hashPath(htSHA256, actualPath); @@ -2511,7 +2509,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() outputRewrites.insert_or_assign( std::string { scratchPath->hashPart() }, std::string { requiredFinalPath.hashPart() }); - rewriteOutput(); + rewriteOutput(outputRewrites); auto narHashAndSize = hashPath(htSHA256, actualPath); ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; newInfo0.narSize = narHashAndSize.second; From d0cecbe87708bd61548afa51fea78ec927403c5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 24 May 2023 15:35:46 +0200 Subject: [PATCH 05/37] Disable the fetchClosure test for old daemons Broken because of the change introduced by #4282 --- tests/fetchClosure.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/fetchClosure.sh b/tests/fetchClosure.sh index a207f647c..21650eb06 100644 --- a/tests/fetchClosure.sh +++ b/tests/fetchClosure.sh @@ -5,6 +5,12 @@ enableFeatures "fetch-closure" clearStore clearCacheCache +# Old daemons don't properly zero out the self-references when +# calculating the CA hashes, so this breaks `nix store +# make-content-addressed` which expects the client and the daemon to +# compute the same hash +requireDaemonNewerThan "2.16.0pre20230524" + # Initialize binary cache. nonCaPath=$(nix build --json --file ./dependencies.nix --no-link | jq -r .[].outputs.out) caPath=$(nix store make-content-addressed --json $nonCaPath | jq -r '.rewrites | map(.) | .[]') From c694f1a2f3773664b08e561c3fe75b5fe97df51f Mon Sep 17 00:00:00 2001 From: Aneesh Agrawal Date: Wed, 24 May 2023 11:05:40 -0400 Subject: [PATCH 06/37] Remove old default from docs for `hashed-mirrors` The `hashed-mirrors` option did use to have this default value, but it was removed and re-added with an empty default value. As the autogenerated docs show the (actual) default values from code, remove this incorrect reference from the docs. I was updating my nix.conf settings after a few years and noticed this. --- src/libstore/globals.hh | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 31dfe5b4e..6d8a0af32 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -896,12 +896,11 @@ public: this, {}, "hashed-mirrors", R"( A list of web servers used by `builtins.fetchurl` to obtain files by - hash. The default is `http://tarballs.nixos.org/`. Given a hash type - *ht* and a base-16 hash *h*, Nix will try to download the file from - *hashed-mirror*/*ht*/*h*. This allows files to be downloaded even if - they have disappeared from their original URI. For example, given - the default mirror `http://tarballs.nixos.org/`, when building the - derivation + hash. Given a hash type *ht* and a base-16 hash *h*, Nix will try to + download the file from *hashed-mirror*/*ht*/*h*. This allows files to + be downloaded even if they have disappeared from their original URI. + For example, given an example mirror `http://tarballs.nixos.org/`, + when building the derivation ```nix builtins.fetchurl { From 38ce40c66c54bfb48104e19e0e874cc047bc2092 Mon Sep 17 00:00:00 2001 From: svenz Date: Thu, 1 Jun 2023 15:20:31 +0200 Subject: [PATCH 07/37] Update install-multi-user.sh delete duplicate word 'with' in status message --- scripts/install-multi-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 79deb2819..c11783158 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -880,7 +880,7 @@ configure_shell_profile() { fi done - task "Setting up shell profiles for Fish with with ${PROFILE_FISH_SUFFIX} inside ${PROFILE_FISH_PREFIXES[*]}" + task "Setting up shell profiles for Fish with ${PROFILE_FISH_SUFFIX} inside ${PROFILE_FISH_PREFIXES[*]}" for fish_prefix in "${PROFILE_FISH_PREFIXES[@]}"; do if [ ! -d "$fish_prefix" ]; then # this specific prefix (ie: /etc/fish) is very likely to exist From b292177eec7e87dcd83fd634b4d8b43de5544c5e Mon Sep 17 00:00:00 2001 From: Shamrock Lee <44064051+ShamrockLee@users.noreply.github.com> Date: Fri, 14 May 2021 01:33:03 +0800 Subject: [PATCH 08/37] Add nix-channel --list-generations Add support to --list-generations as another way to say nix-env --profile /nix/var/nix/profiles/per-user/$USER/channels --list-generations the way we did for nix-channel --rollback [generation id] --- doc/manual/src/command-ref/nix-channel.md | 11 ++++++++++- src/nix-channel/nix-channel.cc | 8 ++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/nix-channel.md b/doc/manual/src/command-ref/nix-channel.md index a210583ae..025f758e7 100644 --- a/doc/manual/src/command-ref/nix-channel.md +++ b/doc/manual/src/command-ref/nix-channel.md @@ -4,7 +4,7 @@ # Synopsis -`nix-channel` {`--add` url [*name*] | `--remove` *name* | `--list` | `--update` [*names…*] | `--rollback` [*generation*] } +`nix-channel` {`--add` url [*name*] | `--remove` *name* | `--list` | `--update` [*names…*] | `--list-generations` | `--rollback` [*generation*] } # Description @@ -39,6 +39,15 @@ This command has the following operations: for `nix-env` operations (by symlinking them from the directory `~/.nix-defexpr`). + - `--list-generations`\ + Prints a list of all the current existing generations for the + channel profile. + + Works the same way as + ``` + nix-env --profile /nix/var/nix/profiles/per-user/$USER/channels --list-generations + ``` + - `--rollback` \[*generation*\]\ Reverts the previous call to `nix-channel --update`. Optionally, you can specify a specific channel generation diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 740737ffe..c1c8edd1d 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -177,6 +177,7 @@ static int main_nix_channel(int argc, char ** argv) cRemove, cList, cUpdate, + cListGenerations, cRollback } cmd = cNone; std::vector args; @@ -193,6 +194,8 @@ static int main_nix_channel(int argc, char ** argv) cmd = cList; } else if (*arg == "--update") { cmd = cUpdate; + } else if (*arg == "--list-generations") { + cmd = cListGenerations; } else if (*arg == "--rollback") { cmd = cRollback; } else { @@ -237,6 +240,11 @@ static int main_nix_channel(int argc, char ** argv) case cUpdate: update(StringSet(args.begin(), args.end())); break; + case cListGenerations: + if (!args.empty()) + throw UsageError("'--list-generations' expects no arguments"); + std::cout << runProgram(settings.nixBinDir + "/nix-env", false, {"--profile", profile, "--list-generations"}) << std::flush; + break; case cRollback: if (args.size() > 1) throw UsageError("'--rollback' has at most one argument"); From 331f0967c4c0b4096c6c34632261a02854f407ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 2 Jun 2023 10:14:53 +0200 Subject: [PATCH 09/37] Add a release note for `nix-channel --list-generations` --- doc/manual/src/release-notes/rl-next.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 78ae99f4b..bde9057c6 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,2 +1,3 @@ # Release X.Y (202?-??-??) +- [`nix-channel`](../command-ref/nix-channel.md) now supports a `--list-generations` subcommand From 0101ce0d9630ee62b2c96800dbcd42fba7c0bf6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 2 Jun 2023 10:20:54 +0200 Subject: [PATCH 10/37] Test `nix-channel --list-generations` Rough test, but the feature is a fairly trivial addition on top of `nix-profile --list-generations`, so it should be enough --- tests/nix-channel.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/nix-channel.sh b/tests/nix-channel.sh index dbb3114f1..b5d935004 100644 --- a/tests/nix-channel.sh +++ b/tests/nix-channel.sh @@ -8,6 +8,7 @@ rm -f $TEST_HOME/.nix-channels $TEST_HOME/.nix-profile nix-channel --add http://foo/bar xyzzy nix-channel --list | grepQuiet http://foo/bar nix-channel --remove xyzzy +[[ $(nix-channel --list-generations | wc -l) == 1 ]] [ -e $TEST_HOME/.nix-channels ] [ "$(cat $TEST_HOME/.nix-channels)" = '' ] @@ -38,6 +39,7 @@ ln -s dependencies.nix $TEST_ROOT/nixexprs/default.nix # Test the update action. nix-channel --add file://$TEST_ROOT/foo nix-channel --update +[[ $(nix-channel --list-generations | wc -l) == 2 ]] # Do a query. nix-env -qa \* --meta --xml --out-path > $TEST_ROOT/meta.xml From 48632e6139889907999a253b36c1786cf085c43f Mon Sep 17 00:00:00 2001 From: Zhaofeng Li Date: Fri, 2 Jun 2023 03:04:13 -0600 Subject: [PATCH 11/37] Also set the PER_LINUX32 personality flag on armv5tel-linux --- src/libstore/build/personality.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libstore/build/personality.cc b/src/libstore/build/personality.cc index 4ad477869..1a6201758 100644 --- a/src/libstore/build/personality.cc +++ b/src/libstore/build/personality.cc @@ -21,7 +21,8 @@ void setPersonality(std::string_view system) && (std::string_view(SYSTEM) == "x86_64-linux" || (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) || system == "armv7l-linux" - || system == "armv6l-linux") + || system == "armv6l-linux" + || system == "armv5tel-linux") { if (personality(PER_LINUX32) == -1) throw SysError("cannot set 32-bit personality"); From b37dd43db4f0c9d675653c7d423407f368298173 Mon Sep 17 00:00:00 2001 From: Emily Trau Date: Mon, 5 Jun 2023 20:18:24 -0700 Subject: [PATCH 12/37] Add missing include `select()` may not be ambiently available for use on every platform --- src/nix/daemon.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index c1a91c63d..9fe9b3b1e 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include From 42e908a3081a040619d5f65ce52d08cb1e29397d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Jun 2023 13:56:43 +0200 Subject: [PATCH 13/37] Upload fallback-paths.nix as part of the release https://github.com/NixOS/nixpkgs/pull/235463 --- maintainers/upload-release.pl | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 07b19c13f..2c6f8ad4f 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -179,6 +179,22 @@ if ($isLatest) { system("docker manifest push nixos/nix:latest") == 0 or die; } +# Upload nix-fallback-paths.nix. +sub getStorePath { + my ($jobName) = @_; + my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json')); + return $buildInfo->{buildoutputs}->{out}->{path} or die "cannot get store path for '$jobName'"; +} + +write_file("$tmpDir/fallback-paths.nix", + "{\n" . + " x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" . + " i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" . + " aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" . + " x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" . + " aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" . + "}\n"); + # Upload release files to S3. for my $fn (glob "$tmpDir/*") { my $name = basename($fn); @@ -199,24 +215,6 @@ for my $fn (glob "$tmpDir/*") { } } -# Print new nix-fallback-paths.nix. -if ($isLatest) { - sub getStorePath { - my ($jobName) = @_; - my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json')); - return $buildInfo->{buildoutputs}->{out}->{path} or die "cannot get store path for '$jobName'"; - } - - print STDERR "nixos/modules/installer/tools/nix-fallback-paths.nix:\n" . - "{\n" . - " x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" . - " i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" . - " aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" . - " x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" . - " aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" . - "}\n"; -} - # Update the "latest" symlink. $channelsBucket->add_key( "nix-latest/install", "", From d14f993a768452e6dfc34e3534aac8878fc711e6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Jun 2023 14:03:24 +0200 Subject: [PATCH 14/37] Fix MIME type --- maintainers/upload-release.pl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 2c6f8ad4f..4263a6cd2 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -205,8 +205,7 @@ for my $fn (glob "$tmpDir/*") { my $configuration = (); $configuration->{content_type} = "application/octet-stream"; - if ($fn =~ /.sha256|install/) { - # Text files + if ($fn =~ /.sha256|install|\.nix$/) { $configuration->{content_type} = "text/plain"; } From 76e032bcf8445dc6db1ebfa1e7a8a9b377dd4242 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 6 Jun 2023 15:37:26 +0200 Subject: [PATCH 15/37] Upload the manual to releases.nixos.org Fixes #8451. --- maintainers/upload-release.pl | 39 +++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 4263a6cd2..ebc536f12 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -80,6 +80,38 @@ my $s3_us = Net::Amazon::S3->new( my $channelsBucket = $s3_us->bucket($channelsBucketName) or die; +sub getStorePath { + my ($jobName, $output) = @_; + my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json')); + return $buildInfo->{buildoutputs}->{$output or "out"}->{path} or die "cannot get store path for '$jobName'"; +} + +sub copyManual { + my $manual = getStorePath("build.x86_64-linux", "doc"); + print "$manual\n"; + + my $manualNar = "$tmpDir/$releaseName-manual.nar.xz"; + print "$manualNar\n"; + + unless (-e $manualNar) { + system("NIX_REMOTE=$binaryCache nix store dump-path '$manual' | xz > '$manualNar'.tmp") == 0 + or die "unable to fetch $manual\n"; + rename("$manualNar.tmp", $manualNar) or die; + } + + unless (-e "$tmpDir/manual") { + system("xz -d < '$manualNar' | nix-store --restore $tmpDir/manual.tmp") == 0 + or die "unable to unpack $manualNar\n"; + rename("$tmpDir/manual.tmp/share/doc/nix/manual", "$tmpDir/manual") or die; + system("rm -rf '$tmpDir/manual.tmp'") == 0 or die; + } + + system("aws s3 sync '$tmpDir/manual' s3://$releasesBucketName/$releaseDir/manual") == 0 + or die "syncing manual to S3\n"; +} + +copyManual; + sub downloadFile { my ($jobName, $productNr, $dstName) = @_; @@ -180,12 +212,6 @@ if ($isLatest) { } # Upload nix-fallback-paths.nix. -sub getStorePath { - my ($jobName) = @_; - my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json')); - return $buildInfo->{buildoutputs}->{out}->{path} or die "cannot get store path for '$jobName'"; -} - write_file("$tmpDir/fallback-paths.nix", "{\n" . " x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" . @@ -198,6 +224,7 @@ write_file("$tmpDir/fallback-paths.nix", # Upload release files to S3. for my $fn (glob "$tmpDir/*") { my $name = basename($fn); + next if $name eq "manual"; my $dstKey = "$releaseDir/" . $name; unless (defined $releasesBucket->head_key($dstKey)) { print STDERR "uploading $fn to s3://$releasesBucketName/$dstKey...\n"; From 3c78920f7358957dba37a379e9d0b062dd3192e2 Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Fri, 9 Jun 2023 17:53:18 +0800 Subject: [PATCH 16/37] Parse TOML timestamps (#8120) Currently `fromTOML` throws an exception when encountering a timestamp since the Nix language lacks a way to represent them. This patch changes this beaviour and makes `fromTOML` parse timestamps as attrsets of the format { _type = "timestamp"; value = "1979-05-27T07:32:00Z"; } This is guarded by an experimental feature flag to leave room for iterating on the representation. --- src/libexpr/primops/fromTOML.cc | 16 ++- src/libutil/experimental-features.cc | 11 +- src/libutil/experimental-features.hh | 1 + tests/lang/eval-fail-fromTOML-timestamps.nix | 130 ++++++++++++++++++ tests/lang/eval-okay-fromTOML-timestamps.exp | 1 + .../lang/eval-okay-fromTOML-timestamps.flags | 1 + tests/lang/eval-okay-fromTOML-timestamps.nix | 130 ++++++++++++++++++ 7 files changed, 286 insertions(+), 4 deletions(-) create mode 100644 tests/lang/eval-fail-fromTOML-timestamps.nix create mode 100644 tests/lang/eval-okay-fromTOML-timestamps.exp create mode 100644 tests/lang/eval-okay-fromTOML-timestamps.flags create mode 100644 tests/lang/eval-okay-fromTOML-timestamps.nix diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 8a5231781..e2a8b3c3a 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -3,6 +3,8 @@ #include "../../toml11/toml.hpp" +#include + namespace nix { static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, Value & val) @@ -58,8 +60,18 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V case toml::value_t::offset_datetime: case toml::value_t::local_date: case toml::value_t::local_time: - // We fail since Nix doesn't have date and time types - throw std::runtime_error("Dates and times are not supported"); + { + if (experimentalFeatureSettings.isEnabled(Xp::ParseTomlTimestamps)) { + auto attrs = state.buildBindings(2); + attrs.alloc("_type").mkString("timestamp"); + std::ostringstream s; + s << t; + attrs.alloc("value").mkString(s.str()); + v.mkAttrs(attrs); + } else { + throw std::runtime_error("Dates and times are not supported"); + } + } break;; case toml::value_t::empty: v.mkNull(); diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 4bf545d7b..c4642d333 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -12,7 +12,7 @@ struct ExperimentalFeatureDetails std::string_view description; }; -constexpr std::array xpFeatureDetails = {{ +constexpr std::array xpFeatureDetails = {{ { .tag = Xp::CaDerivations, .name = "ca-derivations", @@ -214,6 +214,13 @@ constexpr std::array xpFeatureDetails = {{ derivations that are themselves derivations outputs. )", }, + { + .tag = Xp::ParseTomlTimestamps, + .name = "parse-toml-timestamps", + .description = R"( + Allow parsing of timestamps in builtins.fromTOML. + )", + }, }}; static_assert( @@ -248,7 +255,7 @@ std::string_view showExperimentalFeature(const ExperimentalFeature tag) return xpFeatureDetails[(size_t)tag].name; } -nlohmann::json documentExperimentalFeatures() +nlohmann::json documentExperimentalFeatures() { StringMap res; for (auto & xpFeature : xpFeatureDetails) diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 409100592..892c6c371 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -30,6 +30,7 @@ enum struct ExperimentalFeature DiscardReferences, DaemonTrustOverride, DynamicDerivations, + ParseTomlTimestamps, }; /** diff --git a/tests/lang/eval-fail-fromTOML-timestamps.nix b/tests/lang/eval-fail-fromTOML-timestamps.nix new file mode 100644 index 000000000..74cff9470 --- /dev/null +++ b/tests/lang/eval-fail-fromTOML-timestamps.nix @@ -0,0 +1,130 @@ +builtins.fromTOML '' + key = "value" + bare_key = "value" + bare-key = "value" + 1234 = "value" + + "127.0.0.1" = "value" + "character encoding" = "value" + "ʎǝʞ" = "value" + 'key2' = "value" + 'quoted "value"' = "value" + + name = "Orange" + + physical.color = "orange" + physical.shape = "round" + site."google.com" = true + + # This is legal according to the spec, but cpptoml doesn't handle it. + #a.b.c = 1 + #a.d = 2 + + str = "I'm a string. \"You can quote me\". Name\tJos\u00E9\nLocation\tSF." + + int1 = +99 + int2 = 42 + int3 = 0 + int4 = -17 + int5 = 1_000 + int6 = 5_349_221 + int7 = 1_2_3_4_5 + + hex1 = 0xDEADBEEF + hex2 = 0xdeadbeef + hex3 = 0xdead_beef + + oct1 = 0o01234567 + oct2 = 0o755 + + bin1 = 0b11010110 + + flt1 = +1.0 + flt2 = 3.1415 + flt3 = -0.01 + flt4 = 5e+22 + flt5 = 1e6 + flt6 = -2E-2 + flt7 = 6.626e-34 + flt8 = 9_224_617.445_991_228_313 + + bool1 = true + bool2 = false + + odt1 = 1979-05-27T07:32:00Z + odt2 = 1979-05-27T00:32:00-07:00 + odt3 = 1979-05-27T00:32:00.999999-07:00 + odt4 = 1979-05-27 07:32:00Z + ldt1 = 1979-05-27T07:32:00 + ldt2 = 1979-05-27T00:32:00.999999 + ld1 = 1979-05-27 + lt1 = 07:32:00 + lt2 = 00:32:00.999999 + + arr1 = [ 1, 2, 3 ] + arr2 = [ "red", "yellow", "green" ] + arr3 = [ [ 1, 2 ], [3, 4, 5] ] + arr4 = [ "all", 'strings', """are the same""", ''''type''''] + arr5 = [ [ 1, 2 ], ["a", "b", "c"] ] + + arr7 = [ + 1, 2, 3 + ] + + arr8 = [ + 1, + 2, # this is ok + ] + + [table-1] + key1 = "some string" + key2 = 123 + + + [table-2] + key1 = "another string" + key2 = 456 + + [dog."tater.man"] + type.name = "pug" + + [a.b.c] + [ d.e.f ] + [ g . h . i ] + [ j . "ʞ" . 'l' ] + [x.y.z.w] + + name = { first = "Tom", last = "Preston-Werner" } + point = { x = 1, y = 2 } + animal = { type.name = "pug" } + + [[products]] + name = "Hammer" + sku = 738594937 + + [[products]] + + [[products]] + name = "Nail" + sku = 284758393 + color = "gray" + + [[fruit]] + name = "apple" + + [fruit.physical] + color = "red" + shape = "round" + + [[fruit.variety]] + name = "red delicious" + + [[fruit.variety]] + name = "granny smith" + + [[fruit]] + name = "banana" + + [[fruit.variety]] + name = "plantain" +'' diff --git a/tests/lang/eval-okay-fromTOML-timestamps.exp b/tests/lang/eval-okay-fromTOML-timestamps.exp new file mode 100644 index 000000000..08b3c69a6 --- /dev/null +++ b/tests/lang/eval-okay-fromTOML-timestamps.exp @@ -0,0 +1 @@ +{ "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bin1 = 214; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; dog = { "tater.man" = { type = { name = "pug"; }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; fruit = [ { name = "apple"; physical = { color = "red"; shape = "round"; }; variety = [ { name = "red delicious"; } { name = "granny smith"; } ]; } { name = "banana"; variety = [ { name = "plantain"; } ]; } ]; g = { h = { i = { }; }; }; hex1 = 3735928559; hex2 = 3735928559; hex3 = 3735928559; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { l = { }; }; }; key = "value"; key2 = "value"; ld1 = { _type = "timestamp"; value = "1979-05-27"; }; ldt1 = { _type = "timestamp"; value = "1979-05-27T07:32:00"; }; ldt2 = { _type = "timestamp"; value = "1979-05-27T00:32:00.999999"; }; lt1 = { _type = "timestamp"; value = "07:32:00"; }; lt2 = { _type = "timestamp"; value = "00:32:00.999999"; }; name = "Orange"; oct1 = 342391; oct2 = 493; odt1 = { _type = "timestamp"; value = "1979-05-27T07:32:00Z"; }; odt2 = { _type = "timestamp"; value = "1979-05-27T00:32:00-07:00"; }; odt3 = { _type = "timestamp"; value = "1979-05-27T00:32:00.999999-07:00"; }; odt4 = { _type = "timestamp"; value = "1979-05-27T07:32:00Z"; }; physical = { color = "orange"; shape = "round"; }; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; "quoted \"value\"" = "value"; site = { "google.com" = true; }; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { animal = { type = { name = "pug"; }; }; name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } diff --git a/tests/lang/eval-okay-fromTOML-timestamps.flags b/tests/lang/eval-okay-fromTOML-timestamps.flags new file mode 100644 index 000000000..9ed39dc6b --- /dev/null +++ b/tests/lang/eval-okay-fromTOML-timestamps.flags @@ -0,0 +1 @@ +--extra-experimental-features parse-toml-timestamps diff --git a/tests/lang/eval-okay-fromTOML-timestamps.nix b/tests/lang/eval-okay-fromTOML-timestamps.nix new file mode 100644 index 000000000..74cff9470 --- /dev/null +++ b/tests/lang/eval-okay-fromTOML-timestamps.nix @@ -0,0 +1,130 @@ +builtins.fromTOML '' + key = "value" + bare_key = "value" + bare-key = "value" + 1234 = "value" + + "127.0.0.1" = "value" + "character encoding" = "value" + "ʎǝʞ" = "value" + 'key2' = "value" + 'quoted "value"' = "value" + + name = "Orange" + + physical.color = "orange" + physical.shape = "round" + site."google.com" = true + + # This is legal according to the spec, but cpptoml doesn't handle it. + #a.b.c = 1 + #a.d = 2 + + str = "I'm a string. \"You can quote me\". Name\tJos\u00E9\nLocation\tSF." + + int1 = +99 + int2 = 42 + int3 = 0 + int4 = -17 + int5 = 1_000 + int6 = 5_349_221 + int7 = 1_2_3_4_5 + + hex1 = 0xDEADBEEF + hex2 = 0xdeadbeef + hex3 = 0xdead_beef + + oct1 = 0o01234567 + oct2 = 0o755 + + bin1 = 0b11010110 + + flt1 = +1.0 + flt2 = 3.1415 + flt3 = -0.01 + flt4 = 5e+22 + flt5 = 1e6 + flt6 = -2E-2 + flt7 = 6.626e-34 + flt8 = 9_224_617.445_991_228_313 + + bool1 = true + bool2 = false + + odt1 = 1979-05-27T07:32:00Z + odt2 = 1979-05-27T00:32:00-07:00 + odt3 = 1979-05-27T00:32:00.999999-07:00 + odt4 = 1979-05-27 07:32:00Z + ldt1 = 1979-05-27T07:32:00 + ldt2 = 1979-05-27T00:32:00.999999 + ld1 = 1979-05-27 + lt1 = 07:32:00 + lt2 = 00:32:00.999999 + + arr1 = [ 1, 2, 3 ] + arr2 = [ "red", "yellow", "green" ] + arr3 = [ [ 1, 2 ], [3, 4, 5] ] + arr4 = [ "all", 'strings', """are the same""", ''''type''''] + arr5 = [ [ 1, 2 ], ["a", "b", "c"] ] + + arr7 = [ + 1, 2, 3 + ] + + arr8 = [ + 1, + 2, # this is ok + ] + + [table-1] + key1 = "some string" + key2 = 123 + + + [table-2] + key1 = "another string" + key2 = 456 + + [dog."tater.man"] + type.name = "pug" + + [a.b.c] + [ d.e.f ] + [ g . h . i ] + [ j . "ʞ" . 'l' ] + [x.y.z.w] + + name = { first = "Tom", last = "Preston-Werner" } + point = { x = 1, y = 2 } + animal = { type.name = "pug" } + + [[products]] + name = "Hammer" + sku = 738594937 + + [[products]] + + [[products]] + name = "Nail" + sku = 284758393 + color = "gray" + + [[fruit]] + name = "apple" + + [fruit.physical] + color = "red" + shape = "round" + + [[fruit.variety]] + name = "red delicious" + + [[fruit.variety]] + name = "granny smith" + + [[fruit]] + name = "banana" + + [[fruit.variety]] + name = "plantain" +'' From c51f3f1eb250dc006af3de340b1d20b759fed9d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 10 Jun 2023 13:55:58 +0200 Subject: [PATCH 17/37] nix actually needs c++20 now --- doc/manual/src/installation/prerequisites-source.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/installation/prerequisites-source.md b/doc/manual/src/installation/prerequisites-source.md index 5a708f11b..d4babf1ea 100644 --- a/doc/manual/src/installation/prerequisites-source.md +++ b/doc/manual/src/installation/prerequisites-source.md @@ -10,7 +10,7 @@ - Bash Shell. The `./configure` script relies on bashisms, so Bash is required. - - A version of GCC or Clang that supports C++17. + - A version of GCC or Clang that supports C++20. - `pkg-config` to locate dependencies. If your distribution does not provide it, you can get it from From 08089fdd3284fcb27bcec347d6bf88ec2a79a8a0 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Sun, 11 Jun 2023 13:33:38 -0400 Subject: [PATCH 18/37] fix: Do not apply default installables when using --stdin --- src/libcmd/installables.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index a2b882355..684d76a57 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -730,9 +730,9 @@ void RawInstallablesCommand::run(ref store) while (std::cin >> word) { rawInstallables.emplace_back(std::move(word)); } + } else { + applyDefaultInstallables(rawInstallables); } - - applyDefaultInstallables(rawInstallables); run(store, std::move(rawInstallables)); } From 8ec1ba02109eee7e8dfd89f3fbbf060497940469 Mon Sep 17 00:00:00 2001 From: Michal Sojka Date: Sun, 11 Jun 2023 16:57:50 +0200 Subject: [PATCH 19/37] Register all PrimOps via the Info structure This will allow documenting them (in later commits). Note that we keep the old constructor even if it is no longer used by Nix code, because it is used in tests/plugins/plugintest.cc, which suggests that it might be used by some external plugin. --- src/libexpr/primops/context.cc | 30 ++++++++++++++++++++++----- src/libexpr/primops/fetchMercurial.cc | 6 +++++- src/libexpr/primops/fetchTree.cc | 6 +++++- src/libexpr/primops/fromTOML.cc | 6 +++++- 4 files changed, 40 insertions(+), 8 deletions(-) diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 07bf400cf..8ae68e570 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -12,7 +12,11 @@ static void prim_unsafeDiscardStringContext(EvalState & state, const PosIdx pos, v.mkString(*s); } -static RegisterPrimOp primop_unsafeDiscardStringContext("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext); +static RegisterPrimOp primop_unsafeDiscardStringContext({ + .name = "__unsafeDiscardStringContext", + .arity = 1, + .fun = prim_unsafeDiscardStringContext +}); static void prim_hasContext(EvalState & state, const PosIdx pos, Value * * args, Value & v) @@ -22,7 +26,11 @@ static void prim_hasContext(EvalState & state, const PosIdx pos, Value * * args, v.mkBool(!context.empty()); } -static RegisterPrimOp primop_hasContext("__hasContext", 1, prim_hasContext); +static RegisterPrimOp primop_hasContext({ + .name = "__hasContext", + .arity = 1, + .fun = prim_hasContext +}); /* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a @@ -51,7 +59,11 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx p v.mkString(*s, context2); } -static RegisterPrimOp primop_unsafeDiscardOutputDependency("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency); +static RegisterPrimOp primop_unsafeDiscardOutputDependency({ + .name = "__unsafeDiscardOutputDependency", + .arity = 1, + .fun = prim_unsafeDiscardOutputDependency +}); /* Extract the context of a string as a structured Nix value. @@ -119,7 +131,11 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args, v.mkAttrs(attrs); } -static RegisterPrimOp primop_getContext("__getContext", 1, prim_getContext); +static RegisterPrimOp primop_getContext({ + .name = "__getContext", + .arity = 1, + .fun = prim_getContext +}); /* Append the given context to a given string. @@ -192,6 +208,10 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar v.mkString(orig, context); } -static RegisterPrimOp primop_appendContext("__appendContext", 2, prim_appendContext); +static RegisterPrimOp primop_appendContext({ + .name = "__appendContext", + .arity = 2, + .fun = prim_appendContext +}); } diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 2c0d98e74..322692b52 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -88,6 +88,10 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a state.allowPath(tree.storePath); } -static RegisterPrimOp r_fetchMercurial("fetchMercurial", 1, prim_fetchMercurial); +static RegisterPrimOp r_fetchMercurial({ + .name = "fetchMercurial", + .arity = 1, + .fun = prim_fetchMercurial +}); } diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index fe880aaa8..be8159cc8 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -194,7 +194,11 @@ static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args, } // FIXME: document -static RegisterPrimOp primop_fetchTree("fetchTree", 1, prim_fetchTree); +static RegisterPrimOp primop_fetchTree({ + .name = "fetchTree", + .arity = 1, + .fun = prim_fetchTree +}); static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v, const std::string & who, bool unpack, std::string name) diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index e2a8b3c3a..4e6fcdffa 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -90,6 +90,10 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V } } -static RegisterPrimOp primop_fromTOML("fromTOML", 1, prim_fromTOML); +static RegisterPrimOp primop_fromTOML({ + .name = "fromTOML", + .arity = 1, + .fun = prim_fromTOML +}); } From 87c66f6b0f35bc59749cad6bfb3dc2e23de8c86b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 12 Jun 2023 11:10:55 +0200 Subject: [PATCH 20/37] Don't include uds-remote-store.md from a header file Closes #8484. --- src/libstore/uds-remote-store.cc | 8 ++++++++ src/libstore/uds-remote-store.hh | 7 +------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 0fb7c38e9..69dae2da5 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -13,6 +13,14 @@ namespace nix { +std::string UDSRemoteStoreConfig::doc() +{ + return + #include "uds-remote-store.md" + ; +} + + UDSRemoteStore::UDSRemoteStore(const Params & params) : StoreConfig(params) , LocalFSStoreConfig(params) diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index bd1dcb67c..3bbab371c 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -17,12 +17,7 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon const std::string name() override { return "Local Daemon Store"; } - std::string doc() override - { - return - #include "uds-remote-store.md" - ; - } + std::string doc() override; }; class UDSRemoteStore : public virtual UDSRemoteStoreConfig, public virtual LocalFSStore, public virtual RemoteStore From 0e3849dc650250e36fb69c197a8a1fffe4f7d480 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Mon, 12 Jun 2023 08:40:17 -0400 Subject: [PATCH 21/37] test: add test for non-defaulting for stding installable input --- src/libcmd/installables.cc | 2 +- tests/build.sh | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 684d76a57..10b077fb5 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -701,7 +701,7 @@ RawInstallablesCommand::RawInstallablesCommand() { addFlag({ .longName = "stdin", - .description = "Read installables from the standard input.", + .description = "Read installables from the standard input. No default installable applied.", .handler = {&readFromStdIn, true} }); diff --git a/tests/build.sh b/tests/build.sh index 697aff0f9..8ae20f0df 100644 --- a/tests/build.sh +++ b/tests/build.sh @@ -129,3 +129,7 @@ nix build --impure -f multiple-outputs.nix --json e --no-link | jq --exit-status (.drvPath | match(".*multiple-outputs-e.drv")) and (.outputs | keys == ["a_a", "b"])) ' + +# Make sure that `--stdin` works and does not apply any defaults +printf "" | nix build --no-link --stdin --json | jq --exit-status '. == []' +printf "%s\n" "$drv^*" | nix build --no-link --stdin --json | jq --exit-status '.[0]|has("drvPath")' From f961b04484a5770d12ad7866d86d1d1f8e7c687c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jun 2023 22:56:41 +0000 Subject: [PATCH 22/37] Bump zeebe-io/backport-action from 1.3.0 to 1.3.1 Bumps [zeebe-io/backport-action](https://github.com/zeebe-io/backport-action) from 1.3.0 to 1.3.1. - [Release notes](https://github.com/zeebe-io/backport-action/releases) - [Commits](https://github.com/zeebe-io/backport-action/compare/v1.3.0...v1.3.1) --- updated-dependencies: - dependency-name: zeebe-io/backport-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/backport.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 37966bab2..816474ed5 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -21,7 +21,7 @@ jobs: fetch-depth: 0 - name: Create backport PRs # should be kept in sync with `version` - uses: zeebe-io/backport-action@v1.3.0 + uses: zeebe-io/backport-action@v1.3.1 with: # Config README: https://github.com/zeebe-io/backport-action#backport-action github_token: ${{ secrets.GITHUB_TOKEN }} From c6d7c4f9ecb919cced435cf179d8ece5d25b5f06 Mon Sep 17 00:00:00 2001 From: Michal Sojka Date: Sun, 11 Jun 2023 21:38:18 +0200 Subject: [PATCH 23/37] Document fromTOML, hasContext and getContext builtins Until now, these functions were completely missing in the Nix manual. Co-authored-by: Valentin Gagarin --- src/libexpr/primops/context.cc | 28 ++++++++++++++++++++++++++-- src/libexpr/primops/fromTOML.cc | 16 +++++++++++++++- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 8ae68e570..8b3468009 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -28,7 +28,12 @@ static void prim_hasContext(EvalState & state, const PosIdx pos, Value * * args, static RegisterPrimOp primop_hasContext({ .name = "__hasContext", - .arity = 1, + .args = {"s"}, + .doc = R"( + Return `true` if string *s* has a non-empty context. The + context can be obtained with + [`getContext`](#builtins-getContext). + )", .fun = prim_hasContext }); @@ -133,7 +138,26 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args, static RegisterPrimOp primop_getContext({ .name = "__getContext", - .arity = 1, + .args = {"s"}, + .doc = R"( + Return the string context of *s*. + + The string context tracks references to derivations within a string. + It is represented as an attribute set of [store derivation](@docroot@/glossary.md#gloss-store-derivation) paths mapping to output names. + + Using [string interpolation](@docroot@/language/string-interpolation.md) on a derivation will add that derivation to the string context. + For example, + + ```nix + builtins.getContext "${derivation { name = "a"; builder = "b"; system = "c"; }}" + ``` + + evaluates to + + ``` + { "/nix/store/arhvjaf6zmlyn8vh8fgn55rpwnxq0n7l-a.drv" = { outputs = [ "out" ]; }; } + ``` + )", .fun = prim_getContext }); diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 4e6fcdffa..2f4d4022e 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -92,7 +92,21 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V static RegisterPrimOp primop_fromTOML({ .name = "fromTOML", - .arity = 1, + .args = {"e"}, + .doc = R"( + Convert a TOML string to a Nix value. For example, + + ```nix + builtins.fromTOML '' + x=1 + s="a" + [table] + y=2 + '' + ``` + + returns the value `{ s = "a"; table = { y = 2; }; x = 1; }`. + )", .fun = prim_fromTOML }); From bfb5e0bdcddc3ba4866d30c54de49248ca763951 Mon Sep 17 00:00:00 2001 From: scarf Date: Wed, 14 Jun 2023 19:06:04 +0900 Subject: [PATCH 24/37] build: show UID and GID in welcome message --- scripts/install-multi-user.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index c11783158..2e8f2b8c3 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -700,6 +700,10 @@ EOF } welcome_to_nix() { + local -r NIX_UID_RANGES="${NIX_FIRST_BUILD_UID} ~ $((NIX_FIRST_BUILD_UID + NIX_USER_COUNT - 1))" + local -r RANGE_TEXT=$(echo -ne "${BLUE}(uid ${NIX_UID_RANGES})${ESC}") + local -r GROUP_TEXT=$(echo -ne "${BLUE}(gid ${NIX_BUILD_GROUP_ID})${ESC}") + ok "Welcome to the Multi-User Nix Installation" cat < Date: Fri, 24 Apr 2020 17:07:29 +0200 Subject: [PATCH 25/37] darwin installer: remove the file before installing new one Otherwise results into: cp: /Library/LaunchDaemons/org.nixos.nix-daemon.plist and /nix/var/nix/profiles/default/Library/LaunchDaemons/org.nixos.nix-daemon.plist are identical (not copied). --- scripts/install-darwin-multi-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh index 5111a5dde..0326d3415 100644 --- a/scripts/install-darwin-multi-user.sh +++ b/scripts/install-darwin-multi-user.sh @@ -100,7 +100,7 @@ poly_extra_try_me_commands() { poly_configure_nix_daemon_service() { task "Setting up the nix-daemon LaunchDaemon" _sudo "to set up the nix-daemon as a LaunchDaemon" \ - /bin/cp -f "/nix/var/nix/profiles/default$NIX_DAEMON_DEST" "$NIX_DAEMON_DEST" + /usr/bin/install -m -rw-r--r-- "/nix/var/nix/profiles/default$NIX_DAEMON_DEST" "$NIX_DAEMON_DEST" _sudo "to load the LaunchDaemon plist for nix-daemon" \ launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist From 468add5aa0b5b0a686cd07aa5417817bb8799602 Mon Sep 17 00:00:00 2001 From: Daniel Asaturov <122093031+salpelter@users.noreply.github.com> Date: Wed, 14 Jun 2023 22:09:11 +0400 Subject: [PATCH 26/37] Remove dead code (#8504) `filesystem.cc` is the only place where `createSymlink()` is used with three arguments: in the definition of `replaceSymlink()` with three parameters that _is not used at all_. Closes #8495 --- src/libutil/filesystem.cc | 17 +++-------------- src/libutil/util.hh | 6 ++---- 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc index 56be76ecc..11cc0c0e7 100644 --- a/src/libutil/filesystem.cc +++ b/src/libutil/filesystem.cc @@ -63,30 +63,19 @@ std::pair createTempFile(const Path & prefix) return {std::move(fd), tmpl}; } -void createSymlink(const Path & target, const Path & link, - std::optional mtime) +void createSymlink(const Path & target, const Path & link) { if (symlink(target.c_str(), link.c_str())) throw SysError("creating symlink from '%1%' to '%2%'", link, target); - if (mtime) { - struct timeval times[2]; - times[0].tv_sec = *mtime; - times[0].tv_usec = 0; - times[1].tv_sec = *mtime; - times[1].tv_usec = 0; - if (lutimes(link.c_str(), times)) - throw SysError("setting time of symlink '%s'", link); - } } -void replaceSymlink(const Path & target, const Path & link, - std::optional mtime) +void replaceSymlink(const Path & target, const Path & link) { for (unsigned int n = 0; true; n++) { Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); try { - createSymlink(target, tmp, mtime); + createSymlink(target, tmp); } catch (SysError & e) { if (e.errNo == EEXIST) continue; throw; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 00fcb9b79..b302d6f45 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -256,14 +256,12 @@ inline Paths createDirs(PathView path) /** * Create a symlink. */ -void createSymlink(const Path & target, const Path & link, - std::optional mtime = {}); +void createSymlink(const Path & target, const Path & link); /** * Atomically create or replace a symlink. */ -void replaceSymlink(const Path & target, const Path & link, - std::optional mtime = {}); +void replaceSymlink(const Path & target, const Path & link); void renameFile(const Path & src, const Path & dst); From a0c4d585494ab80e9d0095a54cc46eb9e3977f2d Mon Sep 17 00:00:00 2001 From: Michal Sojka Date: Sun, 11 Jun 2023 17:03:14 +0200 Subject: [PATCH 27/37] Remove RegisterPrimOp constructor without support for documentation The remaining constructor RegisterPrimOp::RegisterPrimOp(Info && info) allows specifying the documentation in .args and .doc members of the Info structure. Commit 8ec1ba02109e removed all uses of the removed constructor in the nix binary. Here, we remove the constructor completely as well as its use in a plugin test. According to #8515, we didn't promis to maintain compatibility with external plugins. Fixes #8515 --- src/libexpr/primops.cc | 12 ------------ src/libexpr/primops.hh | 5 ----- tests/plugins/plugintest.cc | 6 +++++- 3 files changed, 5 insertions(+), 18 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 47d8076b1..5b2f7e8b7 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -4058,18 +4058,6 @@ static RegisterPrimOp primop_splitVersion({ RegisterPrimOp::PrimOps * RegisterPrimOp::primOps; -RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun) -{ - if (!primOps) primOps = new PrimOps; - primOps->push_back({ - .name = name, - .args = {}, - .arity = arity, - .fun = fun, - }); -} - - RegisterPrimOp::RegisterPrimOp(Info && info) { if (!primOps) primOps = new PrimOps; diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index 4ae73fe1f..73b7b866c 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -28,11 +28,6 @@ struct RegisterPrimOp * will get called during EvalState initialization, so there * may be primops not yet added and builtins is not yet sorted. */ - RegisterPrimOp( - std::string name, - size_t arity, - PrimOpFun fun); - RegisterPrimOp(Info && info); }; diff --git a/tests/plugins/plugintest.cc b/tests/plugins/plugintest.cc index 04b791021..e02fd68d5 100644 --- a/tests/plugins/plugintest.cc +++ b/tests/plugins/plugintest.cc @@ -21,4 +21,8 @@ static void prim_anotherNull (EvalState & state, const PosIdx pos, Value ** args v.mkBool(false); } -static RegisterPrimOp rp("anotherNull", 0, prim_anotherNull); +static RegisterPrimOp rp({ + .name = "anotherNull", + .arity = 0, + .fun = prim_anotherNull, +}); From a1cf16563f681b5cb3026f2bbca629996ed36d86 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 14 Jun 2023 22:49:58 +0200 Subject: [PATCH 28/37] Fixup description of substituters (#8291) Introduce what substituters "are" in the configuration option entry. Remove arbitrary line breaks for easier editing in the future. Link glossary some more. Co-authored-by: Robert Hensing Co-authored-by: John Ericson --- doc/manual/src/glossary.md | 7 ++++--- src/libstore/globals.hh | 17 ++++++++--------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index e142bd415..47a484826 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -112,9 +112,10 @@ from some server. - [substituter]{#gloss-substituter}\ - A *substituter* is an additional store from which Nix will - copy store objects it doesn't have. For details, see the - [`substituters` option](./command-ref/conf-file.md#conf-substituters). + An additional [store]{#gloss-store} from which Nix can obtain store objects instead of building them. + Often the substituter is a [binary cache](#gloss-binary-cache), but any store can serve as substituter. + + See the [`substituters` configuration option](./command-ref/conf-file.md#conf-substituters) for details. [substituter]: #gloss-substituter diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 940cb48f5..820898350 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -691,20 +691,19 @@ public: Strings{"https://cache.nixos.org/"}, "substituters", R"( - A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) - to be used as substituters, separated by whitespace. - Substituters are tried based on their Priority value, which each substituter can set - independently. Lower value means higher priority. - The default is `https://cache.nixos.org`, with a Priority of 40. + A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) to be used as substituters, separated by whitespace. + A substituter is an additional [store]{@docroot@/glossary.md##gloss-store} from which Nix can obtain [store objects](@docroot@/glossary.md#gloss-store-object) instead of building them. - At least one of the following conditions must be met for Nix to use - a substituter: + Substituters are tried based on their priority value, which each substituter can set independently. + Lower value means higher priority. + The default is `https://cache.nixos.org`, which has a priority of 40. + + At least one of the following conditions must be met for Nix to use a substituter: - the substituter is in the [`trusted-substituters`](#conf-trusted-substituters) list - the user calling Nix is in the [`trusted-users`](#conf-trusted-users) list - In addition, each store path should be trusted as described - in [`trusted-public-keys`](#conf-trusted-public-keys) + In addition, each store path should be trusted as described in [`trusted-public-keys`](#conf-trusted-public-keys) )", {"binary-caches"}}; From ca5752d4faecad8b87ba59892b9ca2a3d8350837 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 16 May 2023 11:21:50 -0400 Subject: [PATCH 29/37] Add another case to the `nix-collect-garbage -d` test --- tests/gc.sh | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/tests/gc.sh b/tests/gc.sh index 95669e25c..9d92ad758 100644 --- a/tests/gc.sh +++ b/tests/gc.sh @@ -56,25 +56,34 @@ testCollectGarbageD () { clearProfiles # Run two `nix-env` commands, should create two generations of # the profile - nix-env -f ./user-envs.nix -i foo-1.0 - nix-env -f ./user-envs.nix -i foo-2.0pre1 - [[ $(nix-env --list-generations | wc -l) -eq 2 ]] + nix-env -f ./user-envs.nix -i foo-1.0 "$@" + nix-env -f ./user-envs.nix -i foo-2.0pre1 "$@" + [[ $(nix-env --list-generations "$@" | wc -l) -eq 2 ]] # Clear the profile history. There should be only one generation # left nix-collect-garbage -d - [[ $(nix-env --list-generations | wc -l) -eq 1 ]] + [[ $(nix-env --list-generations "$@" | wc -l) -eq 1 ]] } + # `nix-env` doesn't work with CA derivations, so let's ignore that bit if we're # using them if [[ -z "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then testCollectGarbageD + # Run the same test, but forcing the profiles an arbitrary location. + rm ~/.nix-profile + ln -s $TEST_ROOT/blah ~/.nix-profile + testCollectGarbageD + # Run the same test, but forcing the profiles at their legacy location under # /nix/var/nix. # + # Note that we *don't* use the default profile; `nix-collect-garbage` will + # need to check the legacy conditional unconditionally not just follow + # `~/.nix-profile` to pass this test. + # # Regression test for #8294 rm ~/.nix-profile - ln -s $NIX_STATE_DIR/profiles/per-user/me ~/.nix-profile - testCollectGarbageD + testCollectGarbageD --profile "$NIX_STATE_DIR/profiles/per-user/me" fi From d4a2ced9cb99253a277c1507baf001d51871842f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 16 May 2023 11:25:45 -0400 Subject: [PATCH 30/37] Split out `nix-collect-garbage -d` test to new file Good for test parallelism, and separation of concerns (core GC vs profiles deleting). --- tests/gc.sh | 37 ------------------------------- tests/local.mk | 1 + tests/nix-collect-garbage-d.sh | 40 ++++++++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 37 deletions(-) create mode 100644 tests/nix-collect-garbage-d.sh diff --git a/tests/gc.sh b/tests/gc.sh index 9d92ad758..ad09a8b39 100644 --- a/tests/gc.sh +++ b/tests/gc.sh @@ -50,40 +50,3 @@ if test -e $outPath/foobar; then false; fi # Check that the store is empty. rmdir $NIX_STORE_DIR/.links rmdir $NIX_STORE_DIR - -## Test `nix-collect-garbage -d` -testCollectGarbageD () { - clearProfiles - # Run two `nix-env` commands, should create two generations of - # the profile - nix-env -f ./user-envs.nix -i foo-1.0 "$@" - nix-env -f ./user-envs.nix -i foo-2.0pre1 "$@" - [[ $(nix-env --list-generations "$@" | wc -l) -eq 2 ]] - - # Clear the profile history. There should be only one generation - # left - nix-collect-garbage -d - [[ $(nix-env --list-generations "$@" | wc -l) -eq 1 ]] -} - -# `nix-env` doesn't work with CA derivations, so let's ignore that bit if we're -# using them -if [[ -z "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then - testCollectGarbageD - - # Run the same test, but forcing the profiles an arbitrary location. - rm ~/.nix-profile - ln -s $TEST_ROOT/blah ~/.nix-profile - testCollectGarbageD - - # Run the same test, but forcing the profiles at their legacy location under - # /nix/var/nix. - # - # Note that we *don't* use the default profile; `nix-collect-garbage` will - # need to check the legacy conditional unconditionally not just follow - # `~/.nix-profile` to pass this test. - # - # Regression test for #8294 - rm ~/.nix-profile - testCollectGarbageD --profile "$NIX_STATE_DIR/profiles/per-user/me" -fi diff --git a/tests/local.mk b/tests/local.mk index 9e340e2e2..2da47d243 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -16,6 +16,7 @@ nix_tests = \ flakes/flake-in-submodule.sh \ ca/gc.sh \ gc.sh \ + nix-collect-garbage-d.sh \ remote-store.sh \ legacy-ssh-store.sh \ lang.sh \ diff --git a/tests/nix-collect-garbage-d.sh b/tests/nix-collect-garbage-d.sh new file mode 100644 index 000000000..bf30f8938 --- /dev/null +++ b/tests/nix-collect-garbage-d.sh @@ -0,0 +1,40 @@ +source common.sh + +clearStore + +## Test `nix-collect-garbage -d` + +# TODO make `nix-env` doesn't work with CA derivations, and make +# `ca/nix-collect-garbage-d.sh` wrapper. + +testCollectGarbageD () { + clearProfiles + # Run two `nix-env` commands, should create two generations of + # the profile + nix-env -f ./user-envs.nix -i foo-1.0 "$@" + nix-env -f ./user-envs.nix -i foo-2.0pre1 "$@" + [[ $(nix-env --list-generations "$@" | wc -l) -eq 2 ]] + + # Clear the profile history. There should be only one generation + # left + nix-collect-garbage -d + [[ $(nix-env --list-generations "$@" | wc -l) -eq 1 ]] +} + +testCollectGarbageD + +# Run the same test, but forcing the profiles an arbitrary location. +rm ~/.nix-profile +ln -s $TEST_ROOT/blah ~/.nix-profile +testCollectGarbageD + +# Run the same test, but forcing the profiles at their legacy location under +# /nix/var/nix. +# +# Note that we *don't* use the default profile; `nix-collect-garbage` will +# need to check the legacy conditional unconditionally not just follow +# `~/.nix-profile` to pass this test. +# +# Regression test for #8294 +rm ~/.nix-profile +testCollectGarbageD --profile "$NIX_STATE_DIR/profiles/per-user/me" From b55f26c65f4ca0e80bd0f3c6436333f054acf6d6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 16 May 2023 12:41:39 -0400 Subject: [PATCH 31/37] Improve `nix-env --delete-generations` docs Co-authored-by: Valentin Gagarin --- .../command-ref/nix-env/delete-generations.md | 59 ++++++++++++++++--- 1 file changed, 50 insertions(+), 9 deletions(-) diff --git a/doc/manual/src/command-ref/nix-env/delete-generations.md b/doc/manual/src/command-ref/nix-env/delete-generations.md index 92cb7f0d9..d828a5b9e 100644 --- a/doc/manual/src/command-ref/nix-env/delete-generations.md +++ b/doc/manual/src/command-ref/nix-env/delete-generations.md @@ -9,14 +9,39 @@ # Description This operation deletes the specified generations of the current profile. -The generations can be a list of generation numbers, the special value -`old` to delete all non-current generations, a value such as `30d` to -delete all generations older than the specified number of days (except -for the generation that was active at that point in time), or a value -such as `+5` to keep the last `5` generations ignoring any newer than -current, e.g., if `30` is the current generation `+5` will delete -generation `25` and all older generations. Periodically deleting old -generations is important to make garbage collection effective. + +*generations* can be a one of the following: + +- `...`:\ + A list of generation numbers, each one a separate command-line argument. + + Delete exactly the profile generations given by their generation number. + Deleting the current generation is not allowed. + +- The special value `old` + + Delete all generations older than the current one. + +- `d`:\ + The last *days* days + + *Example*: `30d` + + Delete all generations older than *days* days. + The generation that was active at that point in time is excluded, and will not be deleted. + +- `+`:\ + The last *count* generations up to the present + + *Example*: `+5` + + Keep the last *count* generations, along with any newer than current. + +Periodically deleting old generations is important to make garbage collection +effective. +The is because profiles are also garbage collection roots — any [store object] reachable from a profile is "alive" and ineligible for deletion. + +[store object]: @docroot@/glossary.md#gloss-store-object {{#include ./opt-common.md}} @@ -28,19 +53,35 @@ generations is important to make garbage collection effective. # Examples +## Delete explicit generation numbers + ```console $ nix-env --delete-generations 3 4 8 ``` +Delete the generations numbered 3, 4, and 8, so long as the current active generation is not any of those. + +## Keep most-recent by count count + ```console $ nix-env --delete-generations +5 ``` +Suppose `30` is the current generation, and we currently have generations numbered `20` through `32`. + +Then this command will delete generations `20` through `25` (`<= 30 - 5`), +and keep generations `26` through `31` (`> 30 - 5`). + +## Keep most-recent in days + ```console $ nix-env --delete-generations 30d ``` +This command will delete all generations older than 30 days, except for the generation that was active 30 days ago (if it currently exists). + +## Delete all older + ```console $ nix-env --profile other_profile --delete-generations old ``` - From 5b7e285727e23135a65597bcaab6d043d6d61ed6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 16 May 2023 10:32:38 -0400 Subject: [PATCH 32/37] Improve `nix-collect-garbage` docs Co-authored-by: Valentin Gagarin --- .../src/command-ref/nix-collect-garbage.md | 67 +++++++++++++++---- 1 file changed, 55 insertions(+), 12 deletions(-) diff --git a/doc/manual/src/command-ref/nix-collect-garbage.md b/doc/manual/src/command-ref/nix-collect-garbage.md index 51db5fc67..a679ceaf7 100644 --- a/doc/manual/src/command-ref/nix-collect-garbage.md +++ b/doc/manual/src/command-ref/nix-collect-garbage.md @@ -1,6 +1,6 @@ # Name -`nix-collect-garbage` - delete unreachable store paths +`nix-collect-garbage` - delete unreachable [store objects] # Synopsis @@ -8,17 +8,57 @@ # Description -The command `nix-collect-garbage` is mostly an alias of [`nix-store ---gc`](@docroot@/command-ref/nix-store/gc.md), that is, it deletes all -unreachable paths in the Nix store to clean up your system. However, -it provides two additional options: `-d` (`--delete-old`), which -deletes all old generations of all profiles in `/nix/var/nix/profiles` -by invoking `nix-env --delete-generations old` on all profiles (of -course, this makes rollbacks to previous configurations impossible); -and `--delete-older-than` *period*, where period is a value such as -`30d`, which deletes all generations older than the specified number -of days in all profiles in `/nix/var/nix/profiles` (except for the -generations that were active at that point in time). +The command `nix-collect-garbage` is mostly an alias of [`nix-store --gc`](@docroot@/command-ref/nix-store/gc.md). +That is, it deletes all unreachable [store objects] in the Nix store to clean up your system. + +However, it provides two additional options, +[`--delete-old`](#opt-delete-old) and [`--delete-older-than`](#opt-delete-older-than), +which also delete old [profiles], allowing potentially more [store objects] to be deleted because profiles are also garbage collection roots. +These options are the equivalent of running +[`nix-env --delete-generations`](@docroot@/command-ref/nix-env/delete-generations.md) +with various augments on multiple profiles, +prior to running `nix-collect-garbage` (or just `nix-store --gc`) without any flags. + +> **Note** +> +> Deleting previous configurations makes rollbacks to them impossible. + +These flags should be used with care, because they potentially delete generations of profiles used by other users on the system. + +## Locations searched for profiles + +`nix-collect-garbage` cannot know about all profiles; that information doesn't exist. +Instead, it looks in a few locations, and acts on all profiles it finds there: + +1. The default profile locations as specified in the [profiles] section of the manual. + +2. > **NOTE** + > + > Not stable; subject to change + > + > Do not rely on this functionality; it just exists for migration purposes and is may change in the future. + > These deprecated paths remain a private implementation detail of Nix. + + `$NIX_STATE_DIR/profiles` and `$NIX_STATE_DIR/profiles/per-user`. + + With the exception of `$NIX_STATE_DIR/profiles/per-user/root` and `$NIX_STATE_DIR/profiles/default`, these directories are no longer used by other commands. + `nix-collect-garbage` looks there anyways in order to clean up profiles from older versions of Nix. + +# Options + +These options are for deleting old [profiles] prior to deleting unreachable [store objects]. + +- [`--delete-old`](#opt-delete-old) / `-d`\ + Delete all old generations of profiles. + + This is the equivalent of invoking `nix-env --delete-generations old` on each found profile. + +- [`--delete-older-than`](#opt-delete-older-than) *period*\ + Delete all generations of profiles older than the specified amount (except for the generations that were active at that point in time). + *period* is a value such as `30d`, which would mean 30 days. + + This is the equivalent of invoking [`nix-env --delete-generations `](@docroot@/command-ref/nix-env/delete-generations.md#generations-days) on each found profile. + See the documentation of that command for additional information about the *period* argument. {{#include ./opt-common.md}} @@ -32,3 +72,6 @@ generations of each profile, do ```console $ nix-collect-garbage -d ``` + +[profiles]: @docroot@/command-ref/files/profiles.md +[store objects]: @docroot@/glossary.md#gloss-store-object From 4b487317c30060024df436d010e6126be3a3d49e Mon Sep 17 00:00:00 2001 From: scarf Date: Thu, 15 Jun 2023 08:52:34 +0900 Subject: [PATCH 33/37] style: use mathematical interval notation --- scripts/install-multi-user.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 2e8f2b8c3..45885bfd9 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -700,8 +700,8 @@ EOF } welcome_to_nix() { - local -r NIX_UID_RANGES="${NIX_FIRST_BUILD_UID} ~ $((NIX_FIRST_BUILD_UID + NIX_USER_COUNT - 1))" - local -r RANGE_TEXT=$(echo -ne "${BLUE}(uid ${NIX_UID_RANGES})${ESC}") + local -r NIX_UID_RANGES="${NIX_FIRST_BUILD_UID}..$((NIX_FIRST_BUILD_UID + NIX_USER_COUNT - 1))" + local -r RANGE_TEXT=$(echo -ne "${BLUE}(uid [${NIX_UID_RANGES}])${ESC}") local -r GROUP_TEXT=$(echo -ne "${BLUE}(gid ${NIX_BUILD_GROUP_ID})${ESC}") ok "Welcome to the Multi-User Nix Installation" From c453719d6e9e1f96f4591b3da359d08b0ce11575 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 15 Jun 2023 02:29:30 +0200 Subject: [PATCH 34/37] rename files referring to antiquotation since we renamed this to string interpolation, file names should be fixed up as well --- ...d-antiquote-1.nix => eval-fail-bad-string-interpolation-1.nix} | 0 ...d-antiquote-2.nix => eval-fail-bad-string-interpolation-2.nix} | 0 ...d-antiquote-3.nix => eval-fail-bad-string-interpolation-3.nix} | 0 ...-antiquotation.exp => eval-okay-path-string-interpolation.exp} | 0 ...-antiquotation.nix => eval-okay-path-string-interpolation.nix} | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename tests/lang/{eval-fail-bad-antiquote-1.nix => eval-fail-bad-string-interpolation-1.nix} (100%) rename tests/lang/{eval-fail-bad-antiquote-2.nix => eval-fail-bad-string-interpolation-2.nix} (100%) rename tests/lang/{eval-fail-bad-antiquote-3.nix => eval-fail-bad-string-interpolation-3.nix} (100%) rename tests/lang/{eval-okay-path-antiquotation.exp => eval-okay-path-string-interpolation.exp} (100%) rename tests/lang/{eval-okay-path-antiquotation.nix => eval-okay-path-string-interpolation.nix} (100%) diff --git a/tests/lang/eval-fail-bad-antiquote-1.nix b/tests/lang/eval-fail-bad-string-interpolation-1.nix similarity index 100% rename from tests/lang/eval-fail-bad-antiquote-1.nix rename to tests/lang/eval-fail-bad-string-interpolation-1.nix diff --git a/tests/lang/eval-fail-bad-antiquote-2.nix b/tests/lang/eval-fail-bad-string-interpolation-2.nix similarity index 100% rename from tests/lang/eval-fail-bad-antiquote-2.nix rename to tests/lang/eval-fail-bad-string-interpolation-2.nix diff --git a/tests/lang/eval-fail-bad-antiquote-3.nix b/tests/lang/eval-fail-bad-string-interpolation-3.nix similarity index 100% rename from tests/lang/eval-fail-bad-antiquote-3.nix rename to tests/lang/eval-fail-bad-string-interpolation-3.nix diff --git a/tests/lang/eval-okay-path-antiquotation.exp b/tests/lang/eval-okay-path-string-interpolation.exp similarity index 100% rename from tests/lang/eval-okay-path-antiquotation.exp rename to tests/lang/eval-okay-path-string-interpolation.exp diff --git a/tests/lang/eval-okay-path-antiquotation.nix b/tests/lang/eval-okay-path-string-interpolation.nix similarity index 100% rename from tests/lang/eval-okay-path-antiquotation.nix rename to tests/lang/eval-okay-path-string-interpolation.nix From 520491607efb7b65704b600077e8fed21562597a Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 15 Jun 2023 02:31:47 +0200 Subject: [PATCH 35/37] docs issue template: move checklist down it's annoying to write issues with the checklist in the way, and the proposal is more important. --- .github/ISSUE_TEMPLATE/missing_documentation.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/missing_documentation.md b/.github/ISSUE_TEMPLATE/missing_documentation.md index 942d7a971..be3f6af97 100644 --- a/.github/ISSUE_TEMPLATE/missing_documentation.md +++ b/.github/ISSUE_TEMPLATE/missing_documentation.md @@ -11,6 +11,10 @@ assignees: '' +## Proposal + + + ## Checklist @@ -22,10 +26,6 @@ assignees: '' [source]: https://github.com/NixOS/nix/tree/master/doc/manual/src [open documentation issues and pull requests]: https://github.com/NixOS/nix/labels/documentation -## Proposal - - - ## Priorities Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc). From 098fbf6273bb5ee69b8dc1a4354d45d9a605541a Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Wed, 14 Jun 2023 21:47:58 -0700 Subject: [PATCH 36/37] src/libexpr/eval.hh: fix typo The option name is `allowed-uris`, not `allowed-uri`. --- src/libexpr/eval.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index d6f4560a5..ea1f5975b 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -741,7 +741,7 @@ struct EvalSettings : Config If set to `true`, the Nix evaluator will not allow access to any files outside of the Nix search path (as set via the `NIX_PATH` environment variable or the `-I` option), or to URIs outside of - `allowed-uri`. The default is `false`. + `allowed-uris`. The default is `false`. )"}; Setting pureEval{this, false, "pure-eval", From 80451b762d71bfbc14dbd2c2d88fda7dcb9571cf Mon Sep 17 00:00:00 2001 From: scarf Date: Thu, 15 Jun 2023 14:47:18 +0900 Subject: [PATCH 37/37] style: use plurals in uid ranges Co-authored-by: John Ericson --- scripts/install-multi-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 45885bfd9..656769d84 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -701,7 +701,7 @@ EOF welcome_to_nix() { local -r NIX_UID_RANGES="${NIX_FIRST_BUILD_UID}..$((NIX_FIRST_BUILD_UID + NIX_USER_COUNT - 1))" - local -r RANGE_TEXT=$(echo -ne "${BLUE}(uid [${NIX_UID_RANGES}])${ESC}") + local -r RANGE_TEXT=$(echo -ne "${BLUE}(uids [${NIX_UID_RANGES}])${ESC}") local -r GROUP_TEXT=$(echo -ne "${BLUE}(gid ${NIX_BUILD_GROUP_ID})${ESC}") ok "Welcome to the Multi-User Nix Installation"