From 3ebe1341abe1b0ad59bd4925517af18d9200f818 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 17 Mar 2023 15:51:08 +0100 Subject: [PATCH 01/48] Make `RewritingSink` accept a map of rewrites Giving it the same semantics as `rewriteStrings`. Also add some tests for it --- src/libexpr/primops.cc | 2 +- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/path-references.cc | 73 +++++++++++++++++ src/libstore/path-references.hh | 25 ++++++ src/{libstore => libutil}/references.cc | 86 +++++---------------- src/{libstore => libutil}/references.hh | 23 +----- src/libutil/tests/references.cc | 46 +++++++++++ 7 files changed, 169 insertions(+), 88 deletions(-) create mode 100644 src/libstore/path-references.cc create mode 100644 src/libstore/path-references.hh rename src/{libstore => libutil}/references.cc (61%) rename src/{libstore => libutil}/references.hh (61%) create mode 100644 src/libutil/tests/references.cc diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index cfae1e5f8..572e76ec6 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -6,7 +6,7 @@ #include "globals.hh" #include "json-to-value.hh" #include "names.hh" -#include "references.hh" +#include "path-references.hh" #include "store-api.hh" #include "util.hh" #include "value-to-json.hh" diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 05d6685da..6f7ab8a3d 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -4,7 +4,7 @@ #include "worker.hh" #include "builtins.hh" #include "builtins/buildenv.hh" -#include "references.hh" +#include "path-references.hh" #include "finally.hh" #include "util.hh" #include "archive.hh" diff --git a/src/libstore/path-references.cc b/src/libstore/path-references.cc new file mode 100644 index 000000000..33cf66ce3 --- /dev/null +++ b/src/libstore/path-references.cc @@ -0,0 +1,73 @@ +#include "path-references.hh" +#include "hash.hh" +#include "util.hh" +#include "archive.hh" + +#include +#include +#include +#include + + +namespace nix { + + +PathRefScanSink::PathRefScanSink(StringSet && hashes, std::map && backMap) + : RefScanSink(std::move(hashes)) + , backMap(std::move(backMap)) +{ } + +PathRefScanSink PathRefScanSink::fromPaths(const StorePathSet & refs) +{ + StringSet hashes; + std::map backMap; + + for (auto & i : refs) { + std::string hashPart(i.hashPart()); + auto inserted = backMap.emplace(hashPart, i).second; + assert(inserted); + hashes.insert(hashPart); + } + + return PathRefScanSink(std::move(hashes), std::move(backMap)); +} + +StorePathSet PathRefScanSink::getResultPaths() +{ + /* Map the hashes found back to their store paths. */ + StorePathSet found; + for (auto & i : getResult()) { + auto j = backMap.find(i); + assert(j != backMap.end()); + found.insert(j->second); + } + + return found; +} + + +std::pair scanForReferences( + const std::string & path, + const StorePathSet & refs) +{ + HashSink hashSink { htSHA256 }; + auto found = scanForReferences(hashSink, path, refs); + auto hash = hashSink.finish(); + return std::pair(found, hash); +} + +StorePathSet scanForReferences( + Sink & toTee, + const Path & path, + const StorePathSet & refs) +{ + PathRefScanSink refsSink = PathRefScanSink::fromPaths(refs); + TeeSink sink { refsSink, toTee }; + + /* Look for the hashes in the NAR dump of the path. */ + dumpPath(path, sink); + + return refsSink.getResultPaths(); +} + +} diff --git a/src/libstore/path-references.hh b/src/libstore/path-references.hh new file mode 100644 index 000000000..7b44e3261 --- /dev/null +++ b/src/libstore/path-references.hh @@ -0,0 +1,25 @@ +#pragma once + +#include "references.hh" +#include "path.hh" + +namespace nix { + +std::pair scanForReferences(const Path & path, const StorePathSet & refs); + +StorePathSet scanForReferences(Sink & toTee, const Path & path, const StorePathSet & refs); + +class PathRefScanSink : public RefScanSink +{ + std::map backMap; + + PathRefScanSink(StringSet && hashes, std::map && backMap); + +public: + + static PathRefScanSink fromPaths(const StorePathSet & refs); + + StorePathSet getResultPaths(); +}; + +} diff --git a/src/libstore/references.cc b/src/libutil/references.cc similarity index 61% rename from src/libstore/references.cc rename to src/libutil/references.cc index 345f4528b..74003584a 100644 --- a/src/libstore/references.cc +++ b/src/libutil/references.cc @@ -6,6 +6,7 @@ #include #include #include +#include namespace nix { @@ -66,69 +67,20 @@ void RefScanSink::operator () (std::string_view data) } -PathRefScanSink::PathRefScanSink(StringSet && hashes, std::map && backMap) - : RefScanSink(std::move(hashes)) - , backMap(std::move(backMap)) -{ } - -PathRefScanSink PathRefScanSink::fromPaths(const StorePathSet & refs) -{ - StringSet hashes; - std::map backMap; - - for (auto & i : refs) { - std::string hashPart(i.hashPart()); - auto inserted = backMap.emplace(hashPart, i).second; - assert(inserted); - hashes.insert(hashPart); - } - - return PathRefScanSink(std::move(hashes), std::move(backMap)); -} - -StorePathSet PathRefScanSink::getResultPaths() -{ - /* Map the hashes found back to their store paths. */ - StorePathSet found; - for (auto & i : getResult()) { - auto j = backMap.find(i); - assert(j != backMap.end()); - found.insert(j->second); - } - - return found; -} - - -std::pair scanForReferences( - const std::string & path, - const StorePathSet & refs) -{ - HashSink hashSink { htSHA256 }; - auto found = scanForReferences(hashSink, path, refs); - auto hash = hashSink.finish(); - return std::pair(found, hash); -} - -StorePathSet scanForReferences( - Sink & toTee, - const Path & path, - const StorePathSet & refs) -{ - PathRefScanSink refsSink = PathRefScanSink::fromPaths(refs); - TeeSink sink { refsSink, toTee }; - - /* Look for the hashes in the NAR dump of the path. */ - dumpPath(path, sink); - - return refsSink.getResultPaths(); -} - - RewritingSink::RewritingSink(const std::string & from, const std::string & to, Sink & nextSink) - : from(from), to(to), nextSink(nextSink) + : RewritingSink({{from, to}}, nextSink) { - assert(from.size() == to.size()); +} + +RewritingSink::RewritingSink(const StringMap & rewrites, Sink & nextSink) + : rewrites(rewrites), nextSink(nextSink) +{ + long unsigned int maxRewriteSize = 0; + for (auto & [from, to] : rewrites) { + assert(from.size() == to.size()); + maxRewriteSize = std::max(maxRewriteSize, from.size()); + } + this->maxRewriteSize = maxRewriteSize; } void RewritingSink::operator () (std::string_view data) @@ -136,13 +88,13 @@ void RewritingSink::operator () (std::string_view data) std::string s(prev); s.append(data); - size_t j = 0; - while ((j = s.find(from, j)) != std::string::npos) { - matches.push_back(pos + j); - s.replace(j, from.size(), to); - } + s = rewriteStrings(s, rewrites); - prev = s.size() < from.size() ? s : std::string(s, s.size() - from.size() + 1, from.size() - 1); + prev = s.size() < maxRewriteSize + ? s + : maxRewriteSize == 0 + ? "" + : std::string(s, s.size() - maxRewriteSize + 1, maxRewriteSize - 1); auto consumed = s.size() - prev.size(); diff --git a/src/libstore/references.hh b/src/libutil/references.hh similarity index 61% rename from src/libstore/references.hh rename to src/libutil/references.hh index 52d71b333..ffd730e7b 100644 --- a/src/libstore/references.hh +++ b/src/libutil/references.hh @@ -2,14 +2,9 @@ ///@file #include "hash.hh" -#include "path.hh" namespace nix { -std::pair scanForReferences(const Path & path, const StorePathSet & refs); - -StorePathSet scanForReferences(Sink & toTee, const Path & path, const StorePathSet & refs); - class RefScanSink : public Sink { StringSet hashes; @@ -28,28 +23,18 @@ public: void operator () (std::string_view data) override; }; -class PathRefScanSink : public RefScanSink -{ - std::map backMap; - - PathRefScanSink(StringSet && hashes, std::map && backMap); - -public: - - static PathRefScanSink fromPaths(const StorePathSet & refs); - - StorePathSet getResultPaths(); -}; - struct RewritingSink : Sink { - std::string from, to, prev; + const StringMap rewrites; + long unsigned int maxRewriteSize; + std::string prev; Sink & nextSink; uint64_t pos = 0; std::vector matches; RewritingSink(const std::string & from, const std::string & to, Sink & nextSink); + RewritingSink(const StringMap & rewrites, Sink & nextSink); void operator () (std::string_view data) override; diff --git a/src/libutil/tests/references.cc b/src/libutil/tests/references.cc new file mode 100644 index 000000000..a517d9aa1 --- /dev/null +++ b/src/libutil/tests/references.cc @@ -0,0 +1,46 @@ +#include "references.hh" +#include + +namespace nix { + +using std::string; + +struct RewriteParams { + string originalString, finalString; + StringMap rewrites; + + friend std::ostream& operator<<(std::ostream& os, const RewriteParams& bar) { + StringSet strRewrites; + for (auto & [from, to] : bar.rewrites) + strRewrites.insert(from + "->" + to); + return os << + "OriginalString: " << bar.originalString << std::endl << + "Rewrites: " << concatStringsSep(",", strRewrites) << std::endl << + "Expected result: " << bar.finalString; + } +}; + +class RewriteTest : public ::testing::TestWithParam { +}; + +TEST_P(RewriteTest, IdentityRewriteIsIdentity) { + RewriteParams param = GetParam(); + StringSink rewritten; + auto rewriter = RewritingSink(param.rewrites, rewritten); + rewriter(param.originalString); + rewriter.flush(); + ASSERT_EQ(rewritten.s, param.finalString); +} + +INSTANTIATE_TEST_CASE_P( + references, + RewriteTest, + ::testing::Values( + RewriteParams{ "foooo", "baroo", {{"foo", "bar"}, {"bar", "baz"}}}, + RewriteParams{ "foooo", "bazoo", {{"fou", "bar"}, {"foo", "baz"}}}, + RewriteParams{ "foooo", "foooo", {}} + ) +); + +} + From a917fb0d53544f7fd29f24d5984702296d457347 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 17 Mar 2023 15:51:08 +0100 Subject: [PATCH 02/48] Use a RewritingSink in derivation goal Possibly this will make it stream --- src/libstore/build/local-derivation-goal.cc | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 6f7ab8a3d..4650b8bae 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2384,13 +2384,16 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() if (!outputRewrites.empty()) { debug("rewriting hashes in '%1%'; cross fingers", actualPath); - /* FIXME: this is in-memory. */ - StringSink sink; - dumpPath(actualPath, sink); + /* FIXME: Is this actually streaming? */ + auto source = sinkToSource([&](Sink & nextSink) { + RewritingSink rsink(outputRewrites, nextSink); + dumpPath(actualPath, rsink); + rsink.flush(); + }); + Path tmpPath = actualPath + ".tmp"; + restorePath(tmpPath, *source); deletePath(actualPath); - sink.s = rewriteStrings(sink.s, outputRewrites); - StringSource source(sink.s); - restorePath(actualPath, source); + movePath(tmpPath, actualPath); /* FIXME: set proper permissions in restorePath() so we don't have to do another traversal. */ From 34e1b464f0054623334355aac94cb1a3e4c35a96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 17 Mar 2023 15:51:08 +0100 Subject: [PATCH 03/48] Normalize the hash-rewriting process when building derivations --- src/libstore/build/local-derivation-goal.cc | 28 ++++++++++----------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 4650b8bae..e9fe1eb73 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2379,14 +2379,14 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() continue; auto references = *referencesOpt; - auto rewriteOutput = [&]() { + auto rewriteOutput = [&](const StringMap & rewrites) { /* Apply hash rewriting if necessary. */ - if (!outputRewrites.empty()) { + if (!rewrites.empty()) { debug("rewriting hashes in '%1%'; cross fingers", actualPath); /* FIXME: Is this actually streaming? */ auto source = sinkToSource([&](Sink & nextSink) { - RewritingSink rsink(outputRewrites, nextSink); + RewritingSink rsink(rewrites, nextSink); dumpPath(actualPath, rsink); rsink.flush(); }); @@ -2442,7 +2442,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() "since recursive hashing is not enabled (one of outputHashMode={flat,text} is true)", actualPath); } - rewriteOutput(); + rewriteOutput(outputRewrites); /* FIXME optimize and deduplicate with addToStore */ std::string oldHashPart { scratchPath->hashPart() }; HashModuloSink caSink { outputHash.hashType, oldHashPart }; @@ -2480,16 +2480,14 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() Hash::dummy, }; if (*scratchPath != newInfo0.path) { - // Also rewrite the output path - auto source = sinkToSource([&](Sink & nextSink) { - RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink); - dumpPath(actualPath, rsink2); - rsink2.flush(); - }); - Path tmpPath = actualPath + ".tmp"; - restorePath(tmpPath, *source); - deletePath(actualPath); - movePath(tmpPath, actualPath); + // If the path has some self-references, we need to rewrite + // them. + // (note that this doesn't invalidate the ca hash we calculated + // above because it's computed *modulo the self-references*, so + // it already takes this rewrite into account). + rewriteOutput( + StringMap{{oldHashPart, + std::string(newInfo0.path.hashPart())}}); } HashResult narHashAndSize = hashPath(htSHA256, actualPath); @@ -2511,7 +2509,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() outputRewrites.insert_or_assign( std::string { scratchPath->hashPart() }, std::string { requiredFinalPath.hashPart() }); - rewriteOutput(); + rewriteOutput(outputRewrites); auto narHashAndSize = hashPath(htSHA256, actualPath); ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; newInfo0.narSize = narHashAndSize.second; From d0cecbe87708bd61548afa51fea78ec927403c5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 24 May 2023 15:35:46 +0200 Subject: [PATCH 04/48] Disable the fetchClosure test for old daemons Broken because of the change introduced by #4282 --- tests/fetchClosure.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/fetchClosure.sh b/tests/fetchClosure.sh index a207f647c..21650eb06 100644 --- a/tests/fetchClosure.sh +++ b/tests/fetchClosure.sh @@ -5,6 +5,12 @@ enableFeatures "fetch-closure" clearStore clearCacheCache +# Old daemons don't properly zero out the self-references when +# calculating the CA hashes, so this breaks `nix store +# make-content-addressed` which expects the client and the daemon to +# compute the same hash +requireDaemonNewerThan "2.16.0pre20230524" + # Initialize binary cache. nonCaPath=$(nix build --json --file ./dependencies.nix --no-link | jq -r .[].outputs.out) caPath=$(nix store make-content-addressed --json $nonCaPath | jq -r '.rewrites | map(.) | .[]') From b292177eec7e87dcd83fd634b4d8b43de5544c5e Mon Sep 17 00:00:00 2001 From: Shamrock Lee <44064051+ShamrockLee@users.noreply.github.com> Date: Fri, 14 May 2021 01:33:03 +0800 Subject: [PATCH 05/48] Add nix-channel --list-generations Add support to --list-generations as another way to say nix-env --profile /nix/var/nix/profiles/per-user/$USER/channels --list-generations the way we did for nix-channel --rollback [generation id] --- doc/manual/src/command-ref/nix-channel.md | 11 ++++++++++- src/nix-channel/nix-channel.cc | 8 ++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/nix-channel.md b/doc/manual/src/command-ref/nix-channel.md index a210583ae..025f758e7 100644 --- a/doc/manual/src/command-ref/nix-channel.md +++ b/doc/manual/src/command-ref/nix-channel.md @@ -4,7 +4,7 @@ # Synopsis -`nix-channel` {`--add` url [*name*] | `--remove` *name* | `--list` | `--update` [*names…*] | `--rollback` [*generation*] } +`nix-channel` {`--add` url [*name*] | `--remove` *name* | `--list` | `--update` [*names…*] | `--list-generations` | `--rollback` [*generation*] } # Description @@ -39,6 +39,15 @@ This command has the following operations: for `nix-env` operations (by symlinking them from the directory `~/.nix-defexpr`). + - `--list-generations`\ + Prints a list of all the current existing generations for the + channel profile. + + Works the same way as + ``` + nix-env --profile /nix/var/nix/profiles/per-user/$USER/channels --list-generations + ``` + - `--rollback` \[*generation*\]\ Reverts the previous call to `nix-channel --update`. Optionally, you can specify a specific channel generation diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 740737ffe..c1c8edd1d 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -177,6 +177,7 @@ static int main_nix_channel(int argc, char ** argv) cRemove, cList, cUpdate, + cListGenerations, cRollback } cmd = cNone; std::vector args; @@ -193,6 +194,8 @@ static int main_nix_channel(int argc, char ** argv) cmd = cList; } else if (*arg == "--update") { cmd = cUpdate; + } else if (*arg == "--list-generations") { + cmd = cListGenerations; } else if (*arg == "--rollback") { cmd = cRollback; } else { @@ -237,6 +240,11 @@ static int main_nix_channel(int argc, char ** argv) case cUpdate: update(StringSet(args.begin(), args.end())); break; + case cListGenerations: + if (!args.empty()) + throw UsageError("'--list-generations' expects no arguments"); + std::cout << runProgram(settings.nixBinDir + "/nix-env", false, {"--profile", profile, "--list-generations"}) << std::flush; + break; case cRollback: if (args.size() > 1) throw UsageError("'--rollback' has at most one argument"); From 331f0967c4c0b4096c6c34632261a02854f407ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 2 Jun 2023 10:14:53 +0200 Subject: [PATCH 06/48] Add a release note for `nix-channel --list-generations` --- doc/manual/src/release-notes/rl-next.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 78ae99f4b..bde9057c6 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,2 +1,3 @@ # Release X.Y (202?-??-??) +- [`nix-channel`](../command-ref/nix-channel.md) now supports a `--list-generations` subcommand From 0101ce0d9630ee62b2c96800dbcd42fba7c0bf6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Fri, 2 Jun 2023 10:20:54 +0200 Subject: [PATCH 07/48] Test `nix-channel --list-generations` Rough test, but the feature is a fairly trivial addition on top of `nix-profile --list-generations`, so it should be enough --- tests/nix-channel.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/nix-channel.sh b/tests/nix-channel.sh index dbb3114f1..b5d935004 100644 --- a/tests/nix-channel.sh +++ b/tests/nix-channel.sh @@ -8,6 +8,7 @@ rm -f $TEST_HOME/.nix-channels $TEST_HOME/.nix-profile nix-channel --add http://foo/bar xyzzy nix-channel --list | grepQuiet http://foo/bar nix-channel --remove xyzzy +[[ $(nix-channel --list-generations | wc -l) == 1 ]] [ -e $TEST_HOME/.nix-channels ] [ "$(cat $TEST_HOME/.nix-channels)" = '' ] @@ -38,6 +39,7 @@ ln -s dependencies.nix $TEST_ROOT/nixexprs/default.nix # Test the update action. nix-channel --add file://$TEST_ROOT/foo nix-channel --update +[[ $(nix-channel --list-generations | wc -l) == 2 ]] # Do a query. nix-env -qa \* --meta --xml --out-path > $TEST_ROOT/meta.xml From db680e0e5751828e788bff4becd89886bde71480 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 8 Jun 2023 02:00:05 +0200 Subject: [PATCH 08/48] refine wording on the purpose of the Nix language packages and configurations are not really a concept in Nix or the Nix language. the idea of transforming files into other files clearly captures what it's all about, and the new phrasing should make the term "derivation" more obvious both in terms of meaning and origin. --- doc/manual/src/language/index.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md index 3eabe1a02..69ebc70c0 100644 --- a/doc/manual/src/language/index.md +++ b/doc/manual/src/language/index.md @@ -4,9 +4,7 @@ The Nix language is - *domain-specific* - It only exists for the Nix package manager: - to describe packages and configurations as well as their variants and compositions. - It is not intended for general purpose use. + Its purpose is to conveniently create and compose precise descriptions of how contents of existing files are used to derive new files. - *declarative* @@ -25,7 +23,7 @@ The Nix language is - *lazy* - Expressions are only evaluated when their value is needed. + Values are only computed when they are needed. - *dynamically typed* From e54538c461e993827d9fbe3b8883d3887f184798 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 9 Jun 2023 16:09:29 +0200 Subject: [PATCH 09/48] restoreMountNamespace(): Restore the original root directory This is necessary when we're in a chroot environment, where the process root is not the same as the root of the mount namespace (e.g. in nixos-enter). Fixes #7602. --- src/libutil/util.cc | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index aa0a154fd..26f9dc8a8 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1853,6 +1853,7 @@ void setStackSize(size_t stackSize) #if __linux__ static AutoCloseFD fdSavedMountNamespace; +static AutoCloseFD fdSavedRoot; #endif void saveMountNamespace() @@ -1860,10 +1861,11 @@ void saveMountNamespace() #if __linux__ static std::once_flag done; std::call_once(done, []() { - AutoCloseFD fd = open("/proc/self/ns/mnt", O_RDONLY); - if (!fd) + fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY); + if (!fdSavedMountNamespace) throw SysError("saving parent mount namespace"); - fdSavedMountNamespace = std::move(fd); + + fdSavedRoot = open("/proc/self/root", O_RDONLY); }); #endif } @@ -1876,9 +1878,16 @@ void restoreMountNamespace() if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) throw SysError("restoring parent mount namespace"); - if (chdir(savedCwd.c_str()) == -1) { - throw SysError("restoring cwd"); + + if (fdSavedRoot) { + if (fchdir(fdSavedRoot.get())) + throw SysError("chdir into saved root"); + if (chroot(".")) + throw SysError("chroot into saved root"); } + + if (chdir(savedCwd.c_str()) == -1) + throw SysError("restoring cwd"); } catch (Error & e) { debug(e.msg()); } From c51f3f1eb250dc006af3de340b1d20b759fed9d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sat, 10 Jun 2023 13:55:58 +0200 Subject: [PATCH 10/48] nix actually needs c++20 now --- doc/manual/src/installation/prerequisites-source.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/installation/prerequisites-source.md b/doc/manual/src/installation/prerequisites-source.md index 5a708f11b..d4babf1ea 100644 --- a/doc/manual/src/installation/prerequisites-source.md +++ b/doc/manual/src/installation/prerequisites-source.md @@ -10,7 +10,7 @@ - Bash Shell. The `./configure` script relies on bashisms, so Bash is required. - - A version of GCC or Clang that supports C++17. + - A version of GCC or Clang that supports C++20. - `pkg-config` to locate dependencies. If your distribution does not provide it, you can get it from From 08089fdd3284fcb27bcec347d6bf88ec2a79a8a0 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Sun, 11 Jun 2023 13:33:38 -0400 Subject: [PATCH 11/48] fix: Do not apply default installables when using --stdin --- src/libcmd/installables.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index a2b882355..684d76a57 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -730,9 +730,9 @@ void RawInstallablesCommand::run(ref store) while (std::cin >> word) { rawInstallables.emplace_back(std::move(word)); } + } else { + applyDefaultInstallables(rawInstallables); } - - applyDefaultInstallables(rawInstallables); run(store, std::move(rawInstallables)); } From 8ec1ba02109eee7e8dfd89f3fbbf060497940469 Mon Sep 17 00:00:00 2001 From: Michal Sojka Date: Sun, 11 Jun 2023 16:57:50 +0200 Subject: [PATCH 12/48] Register all PrimOps via the Info structure This will allow documenting them (in later commits). Note that we keep the old constructor even if it is no longer used by Nix code, because it is used in tests/plugins/plugintest.cc, which suggests that it might be used by some external plugin. --- src/libexpr/primops/context.cc | 30 ++++++++++++++++++++++----- src/libexpr/primops/fetchMercurial.cc | 6 +++++- src/libexpr/primops/fetchTree.cc | 6 +++++- src/libexpr/primops/fromTOML.cc | 6 +++++- 4 files changed, 40 insertions(+), 8 deletions(-) diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 07bf400cf..8ae68e570 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -12,7 +12,11 @@ static void prim_unsafeDiscardStringContext(EvalState & state, const PosIdx pos, v.mkString(*s); } -static RegisterPrimOp primop_unsafeDiscardStringContext("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext); +static RegisterPrimOp primop_unsafeDiscardStringContext({ + .name = "__unsafeDiscardStringContext", + .arity = 1, + .fun = prim_unsafeDiscardStringContext +}); static void prim_hasContext(EvalState & state, const PosIdx pos, Value * * args, Value & v) @@ -22,7 +26,11 @@ static void prim_hasContext(EvalState & state, const PosIdx pos, Value * * args, v.mkBool(!context.empty()); } -static RegisterPrimOp primop_hasContext("__hasContext", 1, prim_hasContext); +static RegisterPrimOp primop_hasContext({ + .name = "__hasContext", + .arity = 1, + .fun = prim_hasContext +}); /* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a @@ -51,7 +59,11 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx p v.mkString(*s, context2); } -static RegisterPrimOp primop_unsafeDiscardOutputDependency("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency); +static RegisterPrimOp primop_unsafeDiscardOutputDependency({ + .name = "__unsafeDiscardOutputDependency", + .arity = 1, + .fun = prim_unsafeDiscardOutputDependency +}); /* Extract the context of a string as a structured Nix value. @@ -119,7 +131,11 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args, v.mkAttrs(attrs); } -static RegisterPrimOp primop_getContext("__getContext", 1, prim_getContext); +static RegisterPrimOp primop_getContext({ + .name = "__getContext", + .arity = 1, + .fun = prim_getContext +}); /* Append the given context to a given string. @@ -192,6 +208,10 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar v.mkString(orig, context); } -static RegisterPrimOp primop_appendContext("__appendContext", 2, prim_appendContext); +static RegisterPrimOp primop_appendContext({ + .name = "__appendContext", + .arity = 2, + .fun = prim_appendContext +}); } diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 2c0d98e74..322692b52 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -88,6 +88,10 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a state.allowPath(tree.storePath); } -static RegisterPrimOp r_fetchMercurial("fetchMercurial", 1, prim_fetchMercurial); +static RegisterPrimOp r_fetchMercurial({ + .name = "fetchMercurial", + .arity = 1, + .fun = prim_fetchMercurial +}); } diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index fe880aaa8..be8159cc8 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -194,7 +194,11 @@ static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args, } // FIXME: document -static RegisterPrimOp primop_fetchTree("fetchTree", 1, prim_fetchTree); +static RegisterPrimOp primop_fetchTree({ + .name = "fetchTree", + .arity = 1, + .fun = prim_fetchTree +}); static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v, const std::string & who, bool unpack, std::string name) diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index e2a8b3c3a..4e6fcdffa 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -90,6 +90,10 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V } } -static RegisterPrimOp primop_fromTOML("fromTOML", 1, prim_fromTOML); +static RegisterPrimOp primop_fromTOML({ + .name = "fromTOML", + .arity = 1, + .fun = prim_fromTOML +}); } From 0e3849dc650250e36fb69c197a8a1fffe4f7d480 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Mon, 12 Jun 2023 08:40:17 -0400 Subject: [PATCH 13/48] test: add test for non-defaulting for stding installable input --- src/libcmd/installables.cc | 2 +- tests/build.sh | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 684d76a57..10b077fb5 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -701,7 +701,7 @@ RawInstallablesCommand::RawInstallablesCommand() { addFlag({ .longName = "stdin", - .description = "Read installables from the standard input.", + .description = "Read installables from the standard input. No default installable applied.", .handler = {&readFromStdIn, true} }); diff --git a/tests/build.sh b/tests/build.sh index 697aff0f9..8ae20f0df 100644 --- a/tests/build.sh +++ b/tests/build.sh @@ -129,3 +129,7 @@ nix build --impure -f multiple-outputs.nix --json e --no-link | jq --exit-status (.drvPath | match(".*multiple-outputs-e.drv")) and (.outputs | keys == ["a_a", "b"])) ' + +# Make sure that `--stdin` works and does not apply any defaults +printf "" | nix build --no-link --stdin --json | jq --exit-status '. == []' +printf "%s\n" "$drv^*" | nix build --no-link --stdin --json | jq --exit-status '.[0]|has("drvPath")' From f961b04484a5770d12ad7866d86d1d1f8e7c687c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 Jun 2023 22:56:41 +0000 Subject: [PATCH 14/48] Bump zeebe-io/backport-action from 1.3.0 to 1.3.1 Bumps [zeebe-io/backport-action](https://github.com/zeebe-io/backport-action) from 1.3.0 to 1.3.1. - [Release notes](https://github.com/zeebe-io/backport-action/releases) - [Commits](https://github.com/zeebe-io/backport-action/compare/v1.3.0...v1.3.1) --- updated-dependencies: - dependency-name: zeebe-io/backport-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/backport.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 37966bab2..816474ed5 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -21,7 +21,7 @@ jobs: fetch-depth: 0 - name: Create backport PRs # should be kept in sync with `version` - uses: zeebe-io/backport-action@v1.3.0 + uses: zeebe-io/backport-action@v1.3.1 with: # Config README: https://github.com/zeebe-io/backport-action#backport-action github_token: ${{ secrets.GITHUB_TOKEN }} From 3402b650cdce318e42acd4dc34f42423cafef587 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 7 Jun 2023 15:06:12 +0200 Subject: [PATCH 15/48] Add a generic check for rev attribute mismatches --- src/libfetchers/fetchers.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 91db3a9eb..2860c1ceb 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -159,6 +159,12 @@ std::pair Input::fetch(ref store) const input.to_string(), *prevLastModified); } + if (auto prevRev = getRev()) { + if (input.getRev() != prevRev) + throw Error("'rev' attribute mismatch in input '%s', expected %s", + input.to_string(), prevRev->gitRev()); + } + if (auto prevRevCount = getRevCount()) { if (input.getRevCount() != prevRevCount) throw Error("'revCount' attribute mismatch in input '%s', expected %d", From 1ad3328c5efea041990fa82e6ad24ae2b4e81c24 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 7 Jun 2023 14:26:30 +0200 Subject: [PATCH 16/48] Allow tarball URLs to redirect to a lockable immutable URL Previously, for tarball flakes, we recorded the original URL of the tarball flake, rather than the URL to which it ultimately redirects. Thus, a flake URL like http://example.org/patchelf-latest.tar that redirects to http://example.org/patchelf-.tar was not really usable. We couldn't record the redirected URL, because sites like GitHub redirect to CDN URLs that we can't rely on to be stable. So now we use the redirected URL only if the server returns the `x-nix-is-immutable` or `x-amz-meta-nix-is-immutable` headers in its response. --- flake.nix | 2 + src/libcmd/common-eval-args.cc | 2 +- src/libexpr/parser.y | 2 +- src/libexpr/primops/fetchTree.cc | 2 +- src/libfetchers/attrs.hh | 1 + src/libfetchers/fetchers.hh | 10 +++- src/libfetchers/github.cc | 10 ++-- src/libfetchers/tarball.cc | 68 +++++++++++++++++++------- src/libstore/filetransfer.cc | 22 +++++++-- src/libstore/filetransfer.hh | 4 ++ tests/nixos/tarball-flakes.nix | 84 ++++++++++++++++++++++++++++++++ 11 files changed, 177 insertions(+), 30 deletions(-) create mode 100644 tests/nixos/tarball-flakes.nix diff --git a/flake.nix b/flake.nix index a4ee80b32..bdbf54169 100644 --- a/flake.nix +++ b/flake.nix @@ -590,6 +590,8 @@ tests.sourcehutFlakes = runNixOSTestFor "x86_64-linux" ./tests/nixos/sourcehut-flakes.nix; + tests.tarballFlakes = runNixOSTestFor "x86_64-linux" ./tests/nixos/tarball-flakes.nix; + tests.containers = runNixOSTestFor "x86_64-linux" ./tests/nixos/containers/containers.nix; tests.setuid = lib.genAttrs diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index ff3abd534..7f97364a1 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -165,7 +165,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s) { if (EvalSettings::isPseudoUrl(s)) { auto storePath = fetchers::downloadTarball( - state.store, EvalSettings::resolvePseudoUrl(s), "source", false).first.storePath; + state.store, EvalSettings::resolvePseudoUrl(s), "source", false).tree.storePath; return state.rootPath(CanonPath(state.store->toRealPath(storePath))); } diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 4d981712a..3b545fd84 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -793,7 +793,7 @@ std::pair EvalState::resolveSearchPathElem(const SearchPathEl if (EvalSettings::isPseudoUrl(elem.second)) { try { auto storePath = fetchers::downloadTarball( - store, EvalSettings::resolvePseudoUrl(elem.second), "source", false).first.storePath; + store, EvalSettings::resolvePseudoUrl(elem.second), "source", false).tree.storePath; res = { true, store->toRealPath(storePath) }; } catch (FileTransferError & e) { logWarning({ diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index fe880aaa8..b34a46fa0 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -262,7 +262,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v // https://github.com/NixOS/nix/issues/4313 auto storePath = unpack - ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).first.storePath + ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).tree.storePath : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath; if (expectedHash) { diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh index 1a14bb023..9f885a793 100644 --- a/src/libfetchers/attrs.hh +++ b/src/libfetchers/attrs.hh @@ -2,6 +2,7 @@ ///@file #include "types.hh" +#include "hash.hh" #include diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 498ad7e4d..d0738f619 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -158,6 +158,7 @@ struct DownloadFileResult StorePath storePath; std::string etag; std::string effectiveUrl; + std::optional immutableUrl; }; DownloadFileResult downloadFile( @@ -167,7 +168,14 @@ DownloadFileResult downloadFile( bool locked, const Headers & headers = {}); -std::pair downloadTarball( +struct DownloadTarballResult +{ + Tree tree; + time_t lastModified; + std::optional immutableUrl; +}; + +DownloadTarballResult downloadTarball( ref store, const std::string & url, const std::string & name, diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 6c1d573ce..80598e7f8 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -207,21 +207,21 @@ struct GitArchiveInputScheme : InputScheme auto url = getDownloadUrl(input); - auto [tree, lastModified] = downloadTarball(store, url.url, input.getName(), true, url.headers); + auto result = downloadTarball(store, url.url, input.getName(), true, url.headers); - input.attrs.insert_or_assign("lastModified", uint64_t(lastModified)); + input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified)); getCache()->add( store, lockedAttrs, { {"rev", rev->gitRev()}, - {"lastModified", uint64_t(lastModified)} + {"lastModified", uint64_t(result.lastModified)} }, - tree.storePath, + result.tree.storePath, true); - return {std::move(tree.storePath), input}; + return {result.tree.storePath, input}; } }; diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 96fe5faca..e42aca6db 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -32,7 +32,8 @@ DownloadFileResult downloadFile( return { .storePath = std::move(cached->storePath), .etag = getStrAttr(cached->infoAttrs, "etag"), - .effectiveUrl = getStrAttr(cached->infoAttrs, "url") + .effectiveUrl = getStrAttr(cached->infoAttrs, "url"), + .immutableUrl = maybeGetStrAttr(cached->infoAttrs, "immutableUrl"), }; }; @@ -55,12 +56,14 @@ DownloadFileResult downloadFile( } // FIXME: write to temporary file. - Attrs infoAttrs({ {"etag", res.etag}, {"url", res.effectiveUri}, }); + if (res.immutableUrl) + infoAttrs.emplace("immutableUrl", *res.immutableUrl); + std::optional storePath; if (res.cached) { @@ -111,10 +114,11 @@ DownloadFileResult downloadFile( .storePath = std::move(*storePath), .etag = res.etag, .effectiveUrl = res.effectiveUri, + .immutableUrl = res.immutableUrl, }; } -std::pair downloadTarball( +DownloadTarballResult downloadTarball( ref store, const std::string & url, const std::string & name, @@ -131,8 +135,9 @@ std::pair downloadTarball( if (cached && !cached->expired) return { - Tree { .actualPath = store->toRealPath(cached->storePath), .storePath = std::move(cached->storePath) }, - getIntAttr(cached->infoAttrs, "lastModified") + .tree = Tree { .actualPath = store->toRealPath(cached->storePath), .storePath = std::move(cached->storePath) }, + .lastModified = (time_t) getIntAttr(cached->infoAttrs, "lastModified"), + .immutableUrl = maybeGetStrAttr(cached->infoAttrs, "immutableUrl"), }; auto res = downloadFile(store, url, name, locked, headers); @@ -160,6 +165,9 @@ std::pair downloadTarball( {"etag", res.etag}, }); + if (res.immutableUrl) + infoAttrs.emplace("immutableUrl", *res.immutableUrl); + getCache()->add( store, inAttrs, @@ -168,8 +176,9 @@ std::pair downloadTarball( locked); return { - Tree { .actualPath = store->toRealPath(*unpackedStorePath), .storePath = std::move(*unpackedStorePath) }, - lastModified, + .tree = Tree { .actualPath = store->toRealPath(*unpackedStorePath), .storePath = std::move(*unpackedStorePath) }, + .lastModified = lastModified, + .immutableUrl = res.immutableUrl, }; } @@ -189,21 +198,33 @@ struct CurlInputScheme : InputScheme virtual bool isValidURL(const ParsedURL & url) const = 0; - std::optional inputFromURL(const ParsedURL & url) const override + std::optional inputFromURL(const ParsedURL & _url) const override { - if (!isValidURL(url)) + if (!isValidURL(_url)) return std::nullopt; Input input; - auto urlWithoutApplicationScheme = url; - urlWithoutApplicationScheme.scheme = parseUrlScheme(url.scheme).transport; + auto url = _url; + + url.scheme = parseUrlScheme(url.scheme).transport; - input.attrs.insert_or_assign("type", inputType()); - input.attrs.insert_or_assign("url", urlWithoutApplicationScheme.to_string()); auto narHash = url.query.find("narHash"); if (narHash != url.query.end()) input.attrs.insert_or_assign("narHash", narHash->second); + + if (auto i = get(url.query, "rev")) + input.attrs.insert_or_assign("rev", *i); + + if (auto i = get(url.query, "revCount")) + if (auto n = string2Int(*i)) + input.attrs.insert_or_assign("revCount", *n); + + url.query.erase("rev"); + url.query.erase("revCount"); + + input.attrs.insert_or_assign("type", inputType()); + input.attrs.insert_or_assign("url", url.to_string()); return input; } @@ -212,7 +233,8 @@ struct CurlInputScheme : InputScheme auto type = maybeGetStrAttr(attrs, "type"); if (type != inputType()) return {}; - std::set allowedNames = {"type", "url", "narHash", "name", "unpack"}; + // FIXME: some of these only apply to TarballInputScheme. + std::set allowedNames = {"type", "url", "narHash", "name", "unpack", "rev", "revCount"}; for (auto & [name, value] : attrs) if (!allowedNames.count(name)) throw Error("unsupported %s input attribute '%s'", *type, name); @@ -275,10 +297,22 @@ struct TarballInputScheme : CurlInputScheme : hasTarballExtension(url.path)); } - std::pair fetch(ref store, const Input & input) override + std::pair fetch(ref store, const Input & _input) override { - auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), input.getName(), false).first; - return {std::move(tree.storePath), input}; + Input input(_input); + auto url = getStrAttr(input.attrs, "url"); + auto result = downloadTarball(store, url, input.getName(), false); + + if (result.immutableUrl) { + auto immutableInput = Input::fromURL(*result.immutableUrl); + // FIXME: would be nice to support arbitrary flakerefs + // here, e.g. git flakes. + if (immutableInput.getType() != "tarball") + throw Error("tarball 'Link' headers that redirect to non-tarball URLs are not supported"); + input = immutableInput; + } + + return {result.tree.storePath, std::move(input)}; } }; diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 2346accbe..38b691279 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -186,9 +186,9 @@ struct curlFileTransfer : public FileTransfer size_t realSize = size * nmemb; std::string line((char *) contents, realSize); printMsg(lvlVomit, "got header for '%s': %s", request.uri, trim(line)); + static std::regex statusLine("HTTP/[^ ]+ +[0-9]+(.*)", std::regex::extended | std::regex::icase); - std::smatch match; - if (std::regex_match(line, match, statusLine)) { + if (std::smatch match; std::regex_match(line, match, statusLine)) { result.etag = ""; result.data.clear(); result.bodySize = 0; @@ -196,9 +196,11 @@ struct curlFileTransfer : public FileTransfer acceptRanges = false; encoding = ""; } else { + auto i = line.find(':'); if (i != std::string::npos) { std::string name = toLower(trim(line.substr(0, i))); + if (name == "etag") { result.etag = trim(line.substr(i + 1)); /* Hack to work around a GitHub bug: it sends @@ -212,10 +214,22 @@ struct curlFileTransfer : public FileTransfer debug("shutting down on 200 HTTP response with expected ETag"); return 0; } - } else if (name == "content-encoding") + } + + else if (name == "content-encoding") encoding = trim(line.substr(i + 1)); + else if (name == "accept-ranges" && toLower(trim(line.substr(i + 1))) == "bytes") acceptRanges = true; + + else if (name == "link" || name == "x-amz-meta-link") { + auto value = trim(line.substr(i + 1)); + static std::regex linkRegex("<([^>]*)>; rel=\"immutable\"", std::regex::extended | std::regex::icase); + if (std::smatch match; std::regex_match(value, match, linkRegex)) + result.immutableUrl = match.str(1); + else + debug("got invalid link header '%s'", value); + } } } return realSize; @@ -345,7 +359,7 @@ struct curlFileTransfer : public FileTransfer { auto httpStatus = getHTTPStatus(); - char * effectiveUriCStr; + char * effectiveUriCStr = nullptr; curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr); if (effectiveUriCStr) result.effectiveUri = effectiveUriCStr; diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index 378c6ff78..a3b0dde1f 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -80,6 +80,10 @@ struct FileTransferResult std::string effectiveUri; std::string data; uint64_t bodySize = 0; + /* An "immutable" URL for this resource (i.e. one whose contents + will never change), as returned by the `Link: ; + rel="immutable"` header. */ + std::optional immutableUrl; }; class Store; diff --git a/tests/nixos/tarball-flakes.nix b/tests/nixos/tarball-flakes.nix new file mode 100644 index 000000000..1d43a5d04 --- /dev/null +++ b/tests/nixos/tarball-flakes.nix @@ -0,0 +1,84 @@ +{ lib, config, nixpkgs, ... }: + +let + pkgs = config.nodes.machine.nixpkgs.pkgs; + + root = pkgs.runCommand "nixpkgs-flake" {} + '' + mkdir -p $out/stable + + set -x + dir=nixpkgs-${nixpkgs.shortRev} + cp -prd ${nixpkgs} $dir + # Set the correct timestamp in the tarball. + find $dir -print0 | xargs -0 touch -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} -- + tar cfz $out/stable/${nixpkgs.rev}.tar.gz $dir --hard-dereference + + echo 'Redirect "/latest.tar.gz" "/stable/${nixpkgs.rev}.tar.gz"' > $out/.htaccess + + echo 'Header set Link "; rel=\"immutable\""' > $out/stable/.htaccess + ''; +in + +{ + name = "tarball-flakes"; + + nodes = + { + machine = + { config, pkgs, ... }: + { networking.firewall.allowedTCPPorts = [ 80 ]; + + services.httpd.enable = true; + services.httpd.adminAddr = "foo@example.org"; + services.httpd.extraConfig = '' + ErrorLog syslog:local6 + ''; + services.httpd.virtualHosts."localhost" = + { servedDirs = + [ { urlPath = "/"; + dir = root; + } + ]; + }; + + virtualisation.writableStore = true; + virtualisation.diskSize = 2048; + virtualisation.additionalPaths = [ pkgs.hello pkgs.fuse ]; + virtualisation.memorySize = 4096; + nix.settings.substituters = lib.mkForce [ ]; + nix.extraOptions = "experimental-features = nix-command flakes"; + }; + }; + + testScript = { nodes }: '' + # fmt: off + import json + + start_all() + + machine.wait_for_unit("httpd.service") + + out = machine.succeed("nix flake metadata --json http://localhost/latest.tar.gz") + print(out) + info = json.loads(out) + + # Check that we got redirected to the immutable URL. + assert info["locked"]["url"] == "http://localhost/stable/${nixpkgs.rev}.tar.gz" + + # Check that we got the rev and revCount attributes. + assert info["revision"] == "${nixpkgs.rev}" + assert info["revCount"] == 1234 + + # Check that fetching with rev/revCount/narHash succeeds. + machine.succeed("nix flake metadata --json http://localhost/latest.tar.gz?rev=" + info["revision"]) + machine.succeed("nix flake metadata --json http://localhost/latest.tar.gz?revCount=" + str(info["revCount"])) + machine.succeed("nix flake metadata --json http://localhost/latest.tar.gz?narHash=" + info["locked"]["narHash"]) + + # Check that fetching fails if we provide incorrect attributes. + machine.fail("nix flake metadata --json http://localhost/latest.tar.gz?rev=493300eb13ae6fb387fbd47bf54a85915acc31c0") + machine.fail("nix flake metadata --json http://localhost/latest.tar.gz?revCount=789") + machine.fail("nix flake metadata --json http://localhost/latest.tar.gz?narHash=sha256-tbudgBSg+bHWHiHnlteNzN8TUvI80ygS9IULh4rklEw=") + ''; + +} From c6d7c4f9ecb919cced435cf179d8ece5d25b5f06 Mon Sep 17 00:00:00 2001 From: Michal Sojka Date: Sun, 11 Jun 2023 21:38:18 +0200 Subject: [PATCH 17/48] Document fromTOML, hasContext and getContext builtins Until now, these functions were completely missing in the Nix manual. Co-authored-by: Valentin Gagarin --- src/libexpr/primops/context.cc | 28 ++++++++++++++++++++++++++-- src/libexpr/primops/fromTOML.cc | 16 +++++++++++++++- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 8ae68e570..8b3468009 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -28,7 +28,12 @@ static void prim_hasContext(EvalState & state, const PosIdx pos, Value * * args, static RegisterPrimOp primop_hasContext({ .name = "__hasContext", - .arity = 1, + .args = {"s"}, + .doc = R"( + Return `true` if string *s* has a non-empty context. The + context can be obtained with + [`getContext`](#builtins-getContext). + )", .fun = prim_hasContext }); @@ -133,7 +138,26 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args, static RegisterPrimOp primop_getContext({ .name = "__getContext", - .arity = 1, + .args = {"s"}, + .doc = R"( + Return the string context of *s*. + + The string context tracks references to derivations within a string. + It is represented as an attribute set of [store derivation](@docroot@/glossary.md#gloss-store-derivation) paths mapping to output names. + + Using [string interpolation](@docroot@/language/string-interpolation.md) on a derivation will add that derivation to the string context. + For example, + + ```nix + builtins.getContext "${derivation { name = "a"; builder = "b"; system = "c"; }}" + ``` + + evaluates to + + ``` + { "/nix/store/arhvjaf6zmlyn8vh8fgn55rpwnxq0n7l-a.drv" = { outputs = [ "out" ]; }; } + ``` + )", .fun = prim_getContext }); diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 4e6fcdffa..2f4d4022e 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -92,7 +92,21 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V static RegisterPrimOp primop_fromTOML({ .name = "fromTOML", - .arity = 1, + .args = {"e"}, + .doc = R"( + Convert a TOML string to a Nix value. For example, + + ```nix + builtins.fromTOML '' + x=1 + s="a" + [table] + y=2 + '' + ``` + + returns the value `{ s = "a"; table = { y = 2; }; x = 1; }`. + )", .fun = prim_fromTOML }); From bfb5e0bdcddc3ba4866d30c54de49248ca763951 Mon Sep 17 00:00:00 2001 From: scarf Date: Wed, 14 Jun 2023 19:06:04 +0900 Subject: [PATCH 18/48] build: show UID and GID in welcome message --- scripts/install-multi-user.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index c11783158..2e8f2b8c3 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -700,6 +700,10 @@ EOF } welcome_to_nix() { + local -r NIX_UID_RANGES="${NIX_FIRST_BUILD_UID} ~ $((NIX_FIRST_BUILD_UID + NIX_USER_COUNT - 1))" + local -r RANGE_TEXT=$(echo -ne "${BLUE}(uid ${NIX_UID_RANGES})${ESC}") + local -r GROUP_TEXT=$(echo -ne "${BLUE}(gid ${NIX_BUILD_GROUP_ID})${ESC}") + ok "Welcome to the Multi-User Nix Installation" cat < Date: Fri, 24 Apr 2020 17:07:29 +0200 Subject: [PATCH 19/48] darwin installer: remove the file before installing new one Otherwise results into: cp: /Library/LaunchDaemons/org.nixos.nix-daemon.plist and /nix/var/nix/profiles/default/Library/LaunchDaemons/org.nixos.nix-daemon.plist are identical (not copied). --- scripts/install-darwin-multi-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh index 5111a5dde..0326d3415 100644 --- a/scripts/install-darwin-multi-user.sh +++ b/scripts/install-darwin-multi-user.sh @@ -100,7 +100,7 @@ poly_extra_try_me_commands() { poly_configure_nix_daemon_service() { task "Setting up the nix-daemon LaunchDaemon" _sudo "to set up the nix-daemon as a LaunchDaemon" \ - /bin/cp -f "/nix/var/nix/profiles/default$NIX_DAEMON_DEST" "$NIX_DAEMON_DEST" + /usr/bin/install -m -rw-r--r-- "/nix/var/nix/profiles/default$NIX_DAEMON_DEST" "$NIX_DAEMON_DEST" _sudo "to load the LaunchDaemon plist for nix-daemon" \ launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist From 468add5aa0b5b0a686cd07aa5417817bb8799602 Mon Sep 17 00:00:00 2001 From: Daniel Asaturov <122093031+salpelter@users.noreply.github.com> Date: Wed, 14 Jun 2023 22:09:11 +0400 Subject: [PATCH 20/48] Remove dead code (#8504) `filesystem.cc` is the only place where `createSymlink()` is used with three arguments: in the definition of `replaceSymlink()` with three parameters that _is not used at all_. Closes #8495 --- src/libutil/filesystem.cc | 17 +++-------------- src/libutil/util.hh | 6 ++---- 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc index 56be76ecc..11cc0c0e7 100644 --- a/src/libutil/filesystem.cc +++ b/src/libutil/filesystem.cc @@ -63,30 +63,19 @@ std::pair createTempFile(const Path & prefix) return {std::move(fd), tmpl}; } -void createSymlink(const Path & target, const Path & link, - std::optional mtime) +void createSymlink(const Path & target, const Path & link) { if (symlink(target.c_str(), link.c_str())) throw SysError("creating symlink from '%1%' to '%2%'", link, target); - if (mtime) { - struct timeval times[2]; - times[0].tv_sec = *mtime; - times[0].tv_usec = 0; - times[1].tv_sec = *mtime; - times[1].tv_usec = 0; - if (lutimes(link.c_str(), times)) - throw SysError("setting time of symlink '%s'", link); - } } -void replaceSymlink(const Path & target, const Path & link, - std::optional mtime) +void replaceSymlink(const Path & target, const Path & link) { for (unsigned int n = 0; true; n++) { Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); try { - createSymlink(target, tmp, mtime); + createSymlink(target, tmp); } catch (SysError & e) { if (e.errNo == EEXIST) continue; throw; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 00fcb9b79..b302d6f45 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -256,14 +256,12 @@ inline Paths createDirs(PathView path) /** * Create a symlink. */ -void createSymlink(const Path & target, const Path & link, - std::optional mtime = {}); +void createSymlink(const Path & target, const Path & link); /** * Atomically create or replace a symlink. */ -void replaceSymlink(const Path & target, const Path & link, - std::optional mtime = {}); +void replaceSymlink(const Path & target, const Path & link); void renameFile(const Path & src, const Path & dst); From a0c4d585494ab80e9d0095a54cc46eb9e3977f2d Mon Sep 17 00:00:00 2001 From: Michal Sojka Date: Sun, 11 Jun 2023 17:03:14 +0200 Subject: [PATCH 21/48] Remove RegisterPrimOp constructor without support for documentation The remaining constructor RegisterPrimOp::RegisterPrimOp(Info && info) allows specifying the documentation in .args and .doc members of the Info structure. Commit 8ec1ba02109e removed all uses of the removed constructor in the nix binary. Here, we remove the constructor completely as well as its use in a plugin test. According to #8515, we didn't promis to maintain compatibility with external plugins. Fixes #8515 --- src/libexpr/primops.cc | 12 ------------ src/libexpr/primops.hh | 5 ----- tests/plugins/plugintest.cc | 6 +++++- 3 files changed, 5 insertions(+), 18 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 47d8076b1..5b2f7e8b7 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -4058,18 +4058,6 @@ static RegisterPrimOp primop_splitVersion({ RegisterPrimOp::PrimOps * RegisterPrimOp::primOps; -RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun) -{ - if (!primOps) primOps = new PrimOps; - primOps->push_back({ - .name = name, - .args = {}, - .arity = arity, - .fun = fun, - }); -} - - RegisterPrimOp::RegisterPrimOp(Info && info) { if (!primOps) primOps = new PrimOps; diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index 4ae73fe1f..73b7b866c 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -28,11 +28,6 @@ struct RegisterPrimOp * will get called during EvalState initialization, so there * may be primops not yet added and builtins is not yet sorted. */ - RegisterPrimOp( - std::string name, - size_t arity, - PrimOpFun fun); - RegisterPrimOp(Info && info); }; diff --git a/tests/plugins/plugintest.cc b/tests/plugins/plugintest.cc index 04b791021..e02fd68d5 100644 --- a/tests/plugins/plugintest.cc +++ b/tests/plugins/plugintest.cc @@ -21,4 +21,8 @@ static void prim_anotherNull (EvalState & state, const PosIdx pos, Value ** args v.mkBool(false); } -static RegisterPrimOp rp("anotherNull", 0, prim_anotherNull); +static RegisterPrimOp rp({ + .name = "anotherNull", + .arity = 0, + .fun = prim_anotherNull, +}); From a1cf16563f681b5cb3026f2bbca629996ed36d86 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 14 Jun 2023 22:49:58 +0200 Subject: [PATCH 22/48] Fixup description of substituters (#8291) Introduce what substituters "are" in the configuration option entry. Remove arbitrary line breaks for easier editing in the future. Link glossary some more. Co-authored-by: Robert Hensing Co-authored-by: John Ericson --- doc/manual/src/glossary.md | 7 ++++--- src/libstore/globals.hh | 17 ++++++++--------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index e142bd415..47a484826 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -112,9 +112,10 @@ from some server. - [substituter]{#gloss-substituter}\ - A *substituter* is an additional store from which Nix will - copy store objects it doesn't have. For details, see the - [`substituters` option](./command-ref/conf-file.md#conf-substituters). + An additional [store]{#gloss-store} from which Nix can obtain store objects instead of building them. + Often the substituter is a [binary cache](#gloss-binary-cache), but any store can serve as substituter. + + See the [`substituters` configuration option](./command-ref/conf-file.md#conf-substituters) for details. [substituter]: #gloss-substituter diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 940cb48f5..820898350 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -691,20 +691,19 @@ public: Strings{"https://cache.nixos.org/"}, "substituters", R"( - A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) - to be used as substituters, separated by whitespace. - Substituters are tried based on their Priority value, which each substituter can set - independently. Lower value means higher priority. - The default is `https://cache.nixos.org`, with a Priority of 40. + A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) to be used as substituters, separated by whitespace. + A substituter is an additional [store]{@docroot@/glossary.md##gloss-store} from which Nix can obtain [store objects](@docroot@/glossary.md#gloss-store-object) instead of building them. - At least one of the following conditions must be met for Nix to use - a substituter: + Substituters are tried based on their priority value, which each substituter can set independently. + Lower value means higher priority. + The default is `https://cache.nixos.org`, which has a priority of 40. + + At least one of the following conditions must be met for Nix to use a substituter: - the substituter is in the [`trusted-substituters`](#conf-trusted-substituters) list - the user calling Nix is in the [`trusted-users`](#conf-trusted-users) list - In addition, each store path should be trusted as described - in [`trusted-public-keys`](#conf-trusted-public-keys) + In addition, each store path should be trusted as described in [`trusted-public-keys`](#conf-trusted-public-keys) )", {"binary-caches"}}; From ca5752d4faecad8b87ba59892b9ca2a3d8350837 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 16 May 2023 11:21:50 -0400 Subject: [PATCH 23/48] Add another case to the `nix-collect-garbage -d` test --- tests/gc.sh | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/tests/gc.sh b/tests/gc.sh index 95669e25c..9d92ad758 100644 --- a/tests/gc.sh +++ b/tests/gc.sh @@ -56,25 +56,34 @@ testCollectGarbageD () { clearProfiles # Run two `nix-env` commands, should create two generations of # the profile - nix-env -f ./user-envs.nix -i foo-1.0 - nix-env -f ./user-envs.nix -i foo-2.0pre1 - [[ $(nix-env --list-generations | wc -l) -eq 2 ]] + nix-env -f ./user-envs.nix -i foo-1.0 "$@" + nix-env -f ./user-envs.nix -i foo-2.0pre1 "$@" + [[ $(nix-env --list-generations "$@" | wc -l) -eq 2 ]] # Clear the profile history. There should be only one generation # left nix-collect-garbage -d - [[ $(nix-env --list-generations | wc -l) -eq 1 ]] + [[ $(nix-env --list-generations "$@" | wc -l) -eq 1 ]] } + # `nix-env` doesn't work with CA derivations, so let's ignore that bit if we're # using them if [[ -z "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then testCollectGarbageD + # Run the same test, but forcing the profiles an arbitrary location. + rm ~/.nix-profile + ln -s $TEST_ROOT/blah ~/.nix-profile + testCollectGarbageD + # Run the same test, but forcing the profiles at their legacy location under # /nix/var/nix. # + # Note that we *don't* use the default profile; `nix-collect-garbage` will + # need to check the legacy conditional unconditionally not just follow + # `~/.nix-profile` to pass this test. + # # Regression test for #8294 rm ~/.nix-profile - ln -s $NIX_STATE_DIR/profiles/per-user/me ~/.nix-profile - testCollectGarbageD + testCollectGarbageD --profile "$NIX_STATE_DIR/profiles/per-user/me" fi From d4a2ced9cb99253a277c1507baf001d51871842f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 16 May 2023 11:25:45 -0400 Subject: [PATCH 24/48] Split out `nix-collect-garbage -d` test to new file Good for test parallelism, and separation of concerns (core GC vs profiles deleting). --- tests/gc.sh | 37 ------------------------------- tests/local.mk | 1 + tests/nix-collect-garbage-d.sh | 40 ++++++++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 37 deletions(-) create mode 100644 tests/nix-collect-garbage-d.sh diff --git a/tests/gc.sh b/tests/gc.sh index 9d92ad758..ad09a8b39 100644 --- a/tests/gc.sh +++ b/tests/gc.sh @@ -50,40 +50,3 @@ if test -e $outPath/foobar; then false; fi # Check that the store is empty. rmdir $NIX_STORE_DIR/.links rmdir $NIX_STORE_DIR - -## Test `nix-collect-garbage -d` -testCollectGarbageD () { - clearProfiles - # Run two `nix-env` commands, should create two generations of - # the profile - nix-env -f ./user-envs.nix -i foo-1.0 "$@" - nix-env -f ./user-envs.nix -i foo-2.0pre1 "$@" - [[ $(nix-env --list-generations "$@" | wc -l) -eq 2 ]] - - # Clear the profile history. There should be only one generation - # left - nix-collect-garbage -d - [[ $(nix-env --list-generations "$@" | wc -l) -eq 1 ]] -} - -# `nix-env` doesn't work with CA derivations, so let's ignore that bit if we're -# using them -if [[ -z "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then - testCollectGarbageD - - # Run the same test, but forcing the profiles an arbitrary location. - rm ~/.nix-profile - ln -s $TEST_ROOT/blah ~/.nix-profile - testCollectGarbageD - - # Run the same test, but forcing the profiles at their legacy location under - # /nix/var/nix. - # - # Note that we *don't* use the default profile; `nix-collect-garbage` will - # need to check the legacy conditional unconditionally not just follow - # `~/.nix-profile` to pass this test. - # - # Regression test for #8294 - rm ~/.nix-profile - testCollectGarbageD --profile "$NIX_STATE_DIR/profiles/per-user/me" -fi diff --git a/tests/local.mk b/tests/local.mk index 9e340e2e2..2da47d243 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -16,6 +16,7 @@ nix_tests = \ flakes/flake-in-submodule.sh \ ca/gc.sh \ gc.sh \ + nix-collect-garbage-d.sh \ remote-store.sh \ legacy-ssh-store.sh \ lang.sh \ diff --git a/tests/nix-collect-garbage-d.sh b/tests/nix-collect-garbage-d.sh new file mode 100644 index 000000000..bf30f8938 --- /dev/null +++ b/tests/nix-collect-garbage-d.sh @@ -0,0 +1,40 @@ +source common.sh + +clearStore + +## Test `nix-collect-garbage -d` + +# TODO make `nix-env` doesn't work with CA derivations, and make +# `ca/nix-collect-garbage-d.sh` wrapper. + +testCollectGarbageD () { + clearProfiles + # Run two `nix-env` commands, should create two generations of + # the profile + nix-env -f ./user-envs.nix -i foo-1.0 "$@" + nix-env -f ./user-envs.nix -i foo-2.0pre1 "$@" + [[ $(nix-env --list-generations "$@" | wc -l) -eq 2 ]] + + # Clear the profile history. There should be only one generation + # left + nix-collect-garbage -d + [[ $(nix-env --list-generations "$@" | wc -l) -eq 1 ]] +} + +testCollectGarbageD + +# Run the same test, but forcing the profiles an arbitrary location. +rm ~/.nix-profile +ln -s $TEST_ROOT/blah ~/.nix-profile +testCollectGarbageD + +# Run the same test, but forcing the profiles at their legacy location under +# /nix/var/nix. +# +# Note that we *don't* use the default profile; `nix-collect-garbage` will +# need to check the legacy conditional unconditionally not just follow +# `~/.nix-profile` to pass this test. +# +# Regression test for #8294 +rm ~/.nix-profile +testCollectGarbageD --profile "$NIX_STATE_DIR/profiles/per-user/me" From b55f26c65f4ca0e80bd0f3c6436333f054acf6d6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 16 May 2023 12:41:39 -0400 Subject: [PATCH 25/48] Improve `nix-env --delete-generations` docs Co-authored-by: Valentin Gagarin --- .../command-ref/nix-env/delete-generations.md | 59 ++++++++++++++++--- 1 file changed, 50 insertions(+), 9 deletions(-) diff --git a/doc/manual/src/command-ref/nix-env/delete-generations.md b/doc/manual/src/command-ref/nix-env/delete-generations.md index 92cb7f0d9..d828a5b9e 100644 --- a/doc/manual/src/command-ref/nix-env/delete-generations.md +++ b/doc/manual/src/command-ref/nix-env/delete-generations.md @@ -9,14 +9,39 @@ # Description This operation deletes the specified generations of the current profile. -The generations can be a list of generation numbers, the special value -`old` to delete all non-current generations, a value such as `30d` to -delete all generations older than the specified number of days (except -for the generation that was active at that point in time), or a value -such as `+5` to keep the last `5` generations ignoring any newer than -current, e.g., if `30` is the current generation `+5` will delete -generation `25` and all older generations. Periodically deleting old -generations is important to make garbage collection effective. + +*generations* can be a one of the following: + +- `...`:\ + A list of generation numbers, each one a separate command-line argument. + + Delete exactly the profile generations given by their generation number. + Deleting the current generation is not allowed. + +- The special value `old` + + Delete all generations older than the current one. + +- `d`:\ + The last *days* days + + *Example*: `30d` + + Delete all generations older than *days* days. + The generation that was active at that point in time is excluded, and will not be deleted. + +- `+`:\ + The last *count* generations up to the present + + *Example*: `+5` + + Keep the last *count* generations, along with any newer than current. + +Periodically deleting old generations is important to make garbage collection +effective. +The is because profiles are also garbage collection roots — any [store object] reachable from a profile is "alive" and ineligible for deletion. + +[store object]: @docroot@/glossary.md#gloss-store-object {{#include ./opt-common.md}} @@ -28,19 +53,35 @@ generations is important to make garbage collection effective. # Examples +## Delete explicit generation numbers + ```console $ nix-env --delete-generations 3 4 8 ``` +Delete the generations numbered 3, 4, and 8, so long as the current active generation is not any of those. + +## Keep most-recent by count count + ```console $ nix-env --delete-generations +5 ``` +Suppose `30` is the current generation, and we currently have generations numbered `20` through `32`. + +Then this command will delete generations `20` through `25` (`<= 30 - 5`), +and keep generations `26` through `31` (`> 30 - 5`). + +## Keep most-recent in days + ```console $ nix-env --delete-generations 30d ``` +This command will delete all generations older than 30 days, except for the generation that was active 30 days ago (if it currently exists). + +## Delete all older + ```console $ nix-env --profile other_profile --delete-generations old ``` - From 5b7e285727e23135a65597bcaab6d043d6d61ed6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 16 May 2023 10:32:38 -0400 Subject: [PATCH 26/48] Improve `nix-collect-garbage` docs Co-authored-by: Valentin Gagarin --- .../src/command-ref/nix-collect-garbage.md | 67 +++++++++++++++---- 1 file changed, 55 insertions(+), 12 deletions(-) diff --git a/doc/manual/src/command-ref/nix-collect-garbage.md b/doc/manual/src/command-ref/nix-collect-garbage.md index 51db5fc67..a679ceaf7 100644 --- a/doc/manual/src/command-ref/nix-collect-garbage.md +++ b/doc/manual/src/command-ref/nix-collect-garbage.md @@ -1,6 +1,6 @@ # Name -`nix-collect-garbage` - delete unreachable store paths +`nix-collect-garbage` - delete unreachable [store objects] # Synopsis @@ -8,17 +8,57 @@ # Description -The command `nix-collect-garbage` is mostly an alias of [`nix-store ---gc`](@docroot@/command-ref/nix-store/gc.md), that is, it deletes all -unreachable paths in the Nix store to clean up your system. However, -it provides two additional options: `-d` (`--delete-old`), which -deletes all old generations of all profiles in `/nix/var/nix/profiles` -by invoking `nix-env --delete-generations old` on all profiles (of -course, this makes rollbacks to previous configurations impossible); -and `--delete-older-than` *period*, where period is a value such as -`30d`, which deletes all generations older than the specified number -of days in all profiles in `/nix/var/nix/profiles` (except for the -generations that were active at that point in time). +The command `nix-collect-garbage` is mostly an alias of [`nix-store --gc`](@docroot@/command-ref/nix-store/gc.md). +That is, it deletes all unreachable [store objects] in the Nix store to clean up your system. + +However, it provides two additional options, +[`--delete-old`](#opt-delete-old) and [`--delete-older-than`](#opt-delete-older-than), +which also delete old [profiles], allowing potentially more [store objects] to be deleted because profiles are also garbage collection roots. +These options are the equivalent of running +[`nix-env --delete-generations`](@docroot@/command-ref/nix-env/delete-generations.md) +with various augments on multiple profiles, +prior to running `nix-collect-garbage` (or just `nix-store --gc`) without any flags. + +> **Note** +> +> Deleting previous configurations makes rollbacks to them impossible. + +These flags should be used with care, because they potentially delete generations of profiles used by other users on the system. + +## Locations searched for profiles + +`nix-collect-garbage` cannot know about all profiles; that information doesn't exist. +Instead, it looks in a few locations, and acts on all profiles it finds there: + +1. The default profile locations as specified in the [profiles] section of the manual. + +2. > **NOTE** + > + > Not stable; subject to change + > + > Do not rely on this functionality; it just exists for migration purposes and is may change in the future. + > These deprecated paths remain a private implementation detail of Nix. + + `$NIX_STATE_DIR/profiles` and `$NIX_STATE_DIR/profiles/per-user`. + + With the exception of `$NIX_STATE_DIR/profiles/per-user/root` and `$NIX_STATE_DIR/profiles/default`, these directories are no longer used by other commands. + `nix-collect-garbage` looks there anyways in order to clean up profiles from older versions of Nix. + +# Options + +These options are for deleting old [profiles] prior to deleting unreachable [store objects]. + +- [`--delete-old`](#opt-delete-old) / `-d`\ + Delete all old generations of profiles. + + This is the equivalent of invoking `nix-env --delete-generations old` on each found profile. + +- [`--delete-older-than`](#opt-delete-older-than) *period*\ + Delete all generations of profiles older than the specified amount (except for the generations that were active at that point in time). + *period* is a value such as `30d`, which would mean 30 days. + + This is the equivalent of invoking [`nix-env --delete-generations `](@docroot@/command-ref/nix-env/delete-generations.md#generations-days) on each found profile. + See the documentation of that command for additional information about the *period* argument. {{#include ./opt-common.md}} @@ -32,3 +72,6 @@ generations of each profile, do ```console $ nix-collect-garbage -d ``` + +[profiles]: @docroot@/command-ref/files/profiles.md +[store objects]: @docroot@/glossary.md#gloss-store-object From 4b487317c30060024df436d010e6126be3a3d49e Mon Sep 17 00:00:00 2001 From: scarf Date: Thu, 15 Jun 2023 08:52:34 +0900 Subject: [PATCH 27/48] style: use mathematical interval notation --- scripts/install-multi-user.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 2e8f2b8c3..45885bfd9 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -700,8 +700,8 @@ EOF } welcome_to_nix() { - local -r NIX_UID_RANGES="${NIX_FIRST_BUILD_UID} ~ $((NIX_FIRST_BUILD_UID + NIX_USER_COUNT - 1))" - local -r RANGE_TEXT=$(echo -ne "${BLUE}(uid ${NIX_UID_RANGES})${ESC}") + local -r NIX_UID_RANGES="${NIX_FIRST_BUILD_UID}..$((NIX_FIRST_BUILD_UID + NIX_USER_COUNT - 1))" + local -r RANGE_TEXT=$(echo -ne "${BLUE}(uid [${NIX_UID_RANGES}])${ESC}") local -r GROUP_TEXT=$(echo -ne "${BLUE}(gid ${NIX_BUILD_GROUP_ID})${ESC}") ok "Welcome to the Multi-User Nix Installation" From c453719d6e9e1f96f4591b3da359d08b0ce11575 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 15 Jun 2023 02:29:30 +0200 Subject: [PATCH 28/48] rename files referring to antiquotation since we renamed this to string interpolation, file names should be fixed up as well --- ...d-antiquote-1.nix => eval-fail-bad-string-interpolation-1.nix} | 0 ...d-antiquote-2.nix => eval-fail-bad-string-interpolation-2.nix} | 0 ...d-antiquote-3.nix => eval-fail-bad-string-interpolation-3.nix} | 0 ...-antiquotation.exp => eval-okay-path-string-interpolation.exp} | 0 ...-antiquotation.nix => eval-okay-path-string-interpolation.nix} | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename tests/lang/{eval-fail-bad-antiquote-1.nix => eval-fail-bad-string-interpolation-1.nix} (100%) rename tests/lang/{eval-fail-bad-antiquote-2.nix => eval-fail-bad-string-interpolation-2.nix} (100%) rename tests/lang/{eval-fail-bad-antiquote-3.nix => eval-fail-bad-string-interpolation-3.nix} (100%) rename tests/lang/{eval-okay-path-antiquotation.exp => eval-okay-path-string-interpolation.exp} (100%) rename tests/lang/{eval-okay-path-antiquotation.nix => eval-okay-path-string-interpolation.nix} (100%) diff --git a/tests/lang/eval-fail-bad-antiquote-1.nix b/tests/lang/eval-fail-bad-string-interpolation-1.nix similarity index 100% rename from tests/lang/eval-fail-bad-antiquote-1.nix rename to tests/lang/eval-fail-bad-string-interpolation-1.nix diff --git a/tests/lang/eval-fail-bad-antiquote-2.nix b/tests/lang/eval-fail-bad-string-interpolation-2.nix similarity index 100% rename from tests/lang/eval-fail-bad-antiquote-2.nix rename to tests/lang/eval-fail-bad-string-interpolation-2.nix diff --git a/tests/lang/eval-fail-bad-antiquote-3.nix b/tests/lang/eval-fail-bad-string-interpolation-3.nix similarity index 100% rename from tests/lang/eval-fail-bad-antiquote-3.nix rename to tests/lang/eval-fail-bad-string-interpolation-3.nix diff --git a/tests/lang/eval-okay-path-antiquotation.exp b/tests/lang/eval-okay-path-string-interpolation.exp similarity index 100% rename from tests/lang/eval-okay-path-antiquotation.exp rename to tests/lang/eval-okay-path-string-interpolation.exp diff --git a/tests/lang/eval-okay-path-antiquotation.nix b/tests/lang/eval-okay-path-string-interpolation.nix similarity index 100% rename from tests/lang/eval-okay-path-antiquotation.nix rename to tests/lang/eval-okay-path-string-interpolation.nix From 520491607efb7b65704b600077e8fed21562597a Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 15 Jun 2023 02:31:47 +0200 Subject: [PATCH 29/48] docs issue template: move checklist down it's annoying to write issues with the checklist in the way, and the proposal is more important. --- .github/ISSUE_TEMPLATE/missing_documentation.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/missing_documentation.md b/.github/ISSUE_TEMPLATE/missing_documentation.md index 942d7a971..be3f6af97 100644 --- a/.github/ISSUE_TEMPLATE/missing_documentation.md +++ b/.github/ISSUE_TEMPLATE/missing_documentation.md @@ -11,6 +11,10 @@ assignees: '' +## Proposal + + + ## Checklist @@ -22,10 +26,6 @@ assignees: '' [source]: https://github.com/NixOS/nix/tree/master/doc/manual/src [open documentation issues and pull requests]: https://github.com/NixOS/nix/labels/documentation -## Proposal - - - ## Priorities Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc). From c3c40763421e881ed5c326ed88a0539b6585c368 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 15 Jun 2023 03:06:38 +0200 Subject: [PATCH 30/48] make domain-specificity more specific also slightly reword the purpose statement to introduce (and explain) derivations right away. --- doc/manual/src/language/index.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md index 69ebc70c0..29950a52d 100644 --- a/doc/manual/src/language/index.md +++ b/doc/manual/src/language/index.md @@ -1,10 +1,11 @@ # Nix Language -The Nix language is +The Nix language is designed for conveniently creating and composing *derivations* – precise descriptions of how contents of existing files are used to derive new files. +It is: - *domain-specific* - Its purpose is to conveniently create and compose precise descriptions of how contents of existing files are used to derive new files. + It comes with [built-in functions](@docroot@/language/builtins.md) to integrate with the Nix store, which manages files and performs the derivations declared in the Nix language. - *declarative* From 098fbf6273bb5ee69b8dc1a4354d45d9a605541a Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Wed, 14 Jun 2023 21:47:58 -0700 Subject: [PATCH 31/48] src/libexpr/eval.hh: fix typo The option name is `allowed-uris`, not `allowed-uri`. --- src/libexpr/eval.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index d6f4560a5..ea1f5975b 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -741,7 +741,7 @@ struct EvalSettings : Config If set to `true`, the Nix evaluator will not allow access to any files outside of the Nix search path (as set via the `NIX_PATH` environment variable or the `-I` option), or to URIs outside of - `allowed-uri`. The default is `false`. + `allowed-uris`. The default is `false`. )"}; Setting pureEval{this, false, "pure-eval", From 80451b762d71bfbc14dbd2c2d88fda7dcb9571cf Mon Sep 17 00:00:00 2001 From: scarf Date: Thu, 15 Jun 2023 14:47:18 +0900 Subject: [PATCH 32/48] style: use plurals in uid ranges Co-authored-by: John Ericson --- scripts/install-multi-user.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 45885bfd9..656769d84 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -701,7 +701,7 @@ EOF welcome_to_nix() { local -r NIX_UID_RANGES="${NIX_FIRST_BUILD_UID}..$((NIX_FIRST_BUILD_UID + NIX_USER_COUNT - 1))" - local -r RANGE_TEXT=$(echo -ne "${BLUE}(uid [${NIX_UID_RANGES}])${ESC}") + local -r RANGE_TEXT=$(echo -ne "${BLUE}(uids [${NIX_UID_RANGES}])${ESC}") local -r GROUP_TEXT=$(echo -ne "${BLUE}(gid ${NIX_BUILD_GROUP_ID})${ESC}") ok "Welcome to the Multi-User Nix Installation" From d2696cdd1ee404ef6c6bd532cb90007dd8ee3e9f Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 14 Jun 2023 22:57:42 +0200 Subject: [PATCH 33/48] Fix build hook error for libstore library users A library shouldn't require changes to the caller's argument handling, especially if it doesn't have to, and indeed we don't have to. This changes the lookup order to prioritize the hardcoded path to nix if it exists. The static executable still finds itself through /proc and the like. --- .gitignore | 1 + Makefile | 1 + src/libstore/globals.cc | 25 ++++++++++++++- tests/local.mk | 2 ++ tests/test-libstoreconsumer.sh | 6 ++++ tests/test-libstoreconsumer/README.md | 6 ++++ tests/test-libstoreconsumer/local.mk | 12 +++++++ tests/test-libstoreconsumer/main.cc | 45 +++++++++++++++++++++++++++ 8 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 tests/test-libstoreconsumer.sh create mode 100644 tests/test-libstoreconsumer/README.md create mode 100644 tests/test-libstoreconsumer/local.mk create mode 100644 tests/test-libstoreconsumer/main.cc diff --git a/.gitignore b/.gitignore index 7ae1071d0..29d9106ae 100644 --- a/.gitignore +++ b/.gitignore @@ -89,6 +89,7 @@ perl/Makefile.config /tests/ca/config.nix /tests/dyn-drv/config.nix /tests/repl-result-out +/tests/test-libstoreconsumer/test-libstoreconsumer # /tests/lang/ /tests/lang/*.out diff --git a/Makefile b/Makefile index d6b49473a..c6220482a 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,7 @@ makefiles += \ src/libstore/tests/local.mk \ src/libexpr/tests/local.mk \ tests/local.mk \ + tests/test-libstoreconsumer/local.mk \ tests/plugins/local.mk else makefiles += \ diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 32e9a6ea9..d53377239 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -77,7 +77,30 @@ Settings::Settings() allowedImpureHostPrefixes = tokenizeString("/System/Library /usr/lib /dev /bin/sh"); #endif - buildHook = getSelfExe().value_or("nix") + " __build-remote"; + /* Set the build hook location + + For builds we perform a self-invocation, so Nix has to be self-aware. + That is, it has to know where it is installed. We don't think it's sentient. + + Normally, nix is installed according to `nixBinDir`, which is set at compile time, + but can be overridden. This makes for a great default that works even if this + code is linked as a library into some other program whose main is not aware + that it might need to be a build remote hook. + + However, it may not have been installed at all. For example, if it's a static build, + there's a good chance that it has been moved out of its installation directory. + That makes `nixBinDir` useless. Instead, we'll query the OS for the path to the + current executable, using `getSelfExe()`. + + As a last resort, we resort to `PATH`. Hopefully we find a `nix` there that's compatible. + If you're porting Nix to a new platform, that might be good enough for a while, but + you'll want to improve `getSelfExe()` to work on your platform. + */ + std::string nixExePath = nixBinDir + "/nix"; + if (!pathExists(nixExePath)) { + nixExePath = getSelfExe().value_or("nix"); + } + buildHook = nixExePath + " __build-remote"; } void loadConfFile() diff --git a/tests/local.mk b/tests/local.mk index 9e340e2e2..a37365efe 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -134,6 +134,7 @@ nix_tests = \ flakes/show.sh \ impure-derivations.sh \ path-from-hash-part.sh \ + test-libstoreconsumer.sh \ toString-path.sh ifeq ($(HAVE_LIBCPUID), 1) @@ -152,6 +153,7 @@ test-deps += \ tests/common/vars-and-functions.sh \ tests/config.nix \ tests/ca/config.nix \ + tests/test-libstoreconsumer/test-libstoreconsumer \ tests/dyn-drv/config.nix ifeq ($(BUILD_SHARED_LIBS), 1) diff --git a/tests/test-libstoreconsumer.sh b/tests/test-libstoreconsumer.sh new file mode 100644 index 000000000..8a77cf5a1 --- /dev/null +++ b/tests/test-libstoreconsumer.sh @@ -0,0 +1,6 @@ +source common.sh + +drv="$(nix-instantiate simple.nix)" +cat "$drv" +out="$(./test-libstoreconsumer/test-libstoreconsumer "$drv")" +cat "$out/hello" | grep -F "Hello World!" diff --git a/tests/test-libstoreconsumer/README.md b/tests/test-libstoreconsumer/README.md new file mode 100644 index 000000000..ded69850f --- /dev/null +++ b/tests/test-libstoreconsumer/README.md @@ -0,0 +1,6 @@ + +A very simple C++ consumer of the libstore library. + + - Keep it simple. Library consumers expect something simple. + - No build hook, or any other reinvocations. + - No more global state than necessary. diff --git a/tests/test-libstoreconsumer/local.mk b/tests/test-libstoreconsumer/local.mk new file mode 100644 index 000000000..cd2d0c7f8 --- /dev/null +++ b/tests/test-libstoreconsumer/local.mk @@ -0,0 +1,12 @@ +programs += test-libstoreconsumer + +test-libstoreconsumer_DIR := $(d) + +test-libstoreconsumer_SOURCES := \ + $(wildcard $(d)/*.cc) \ + +test-libstoreconsumer_CXXFLAGS += -I src/libutil -I src/libstore + +test-libstoreconsumer_LIBS = libstore libutil + +test-libstoreconsumer_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) $(LOWDOWN_LIBS) diff --git a/tests/test-libstoreconsumer/main.cc b/tests/test-libstoreconsumer/main.cc new file mode 100644 index 000000000..31b6d8ef1 --- /dev/null +++ b/tests/test-libstoreconsumer/main.cc @@ -0,0 +1,45 @@ +#include "globals.hh" +#include "store-api.hh" +#include "build-result.hh" +#include + +using namespace nix; + +int main (int argc, char **argv) +{ + try { + if (argc != 2) { + std::cerr << "Usage: " << argv[0] << " store/path/to/something.drv\n"; + return 1; + } + + std::string drvPath = argv[1]; + + initLibStore(); + + auto store = nix::openStore(); + + // build the derivation + + std::vector paths { + DerivedPath::Built { + .drvPath = store->parseStorePath(drvPath), + .outputs = OutputsSpec::Names{"out"} + } + }; + + const auto results = store->buildPathsWithResults(paths, bmNormal, store); + + for (const auto & result : results) { + for (const auto & [outputName, realisation] : result.builtOutputs) { + std::cout << store->printStorePath(realisation.outPath) << "\n"; + } + } + + return 0; + + } catch (const std::exception & e) { + std::cerr << "Error: " << e.what() << "\n"; + return 1; + } +} From b2247ef4f6b02cf4e666455eb9afc86398fff35d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 15 Jun 2023 21:24:14 +0200 Subject: [PATCH 34/48] Don't assume the type of string::size_type The code accidentally conflated `std::string::size_type` and `long unsigned int`. This was fine on 64bits machines where they are apparently the same in practice, but not on 32bits. Fix that by using `std::string::size_type` everywhere. --- src/libutil/references.cc | 2 +- src/libutil/references.hh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/references.cc b/src/libutil/references.cc index 74003584a..7f59b4c09 100644 --- a/src/libutil/references.cc +++ b/src/libutil/references.cc @@ -75,7 +75,7 @@ RewritingSink::RewritingSink(const std::string & from, const std::string & to, S RewritingSink::RewritingSink(const StringMap & rewrites, Sink & nextSink) : rewrites(rewrites), nextSink(nextSink) { - long unsigned int maxRewriteSize = 0; + std::string::size_type maxRewriteSize = 0; for (auto & [from, to] : rewrites) { assert(from.size() == to.size()); maxRewriteSize = std::max(maxRewriteSize, from.size()); diff --git a/src/libutil/references.hh b/src/libutil/references.hh index ffd730e7b..f0baeffe1 100644 --- a/src/libutil/references.hh +++ b/src/libutil/references.hh @@ -26,7 +26,7 @@ public: struct RewritingSink : Sink { const StringMap rewrites; - long unsigned int maxRewriteSize; + std::string::size_type maxRewriteSize; std::string prev; Sink & nextSink; uint64_t pos = 0; From cab03fb7794f6e20e02dc1460a78201ab3955c18 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 16 Jun 2023 15:58:42 +0200 Subject: [PATCH 35/48] Add docs --- doc/manual/src/SUMMARY.md.in | 2 ++ doc/manual/src/protocols/protocols.md | 4 +++ doc/manual/src/protocols/tarball-fetcher.md | 40 +++++++++++++++++++++ 3 files changed, 46 insertions(+) create mode 100644 doc/manual/src/protocols/protocols.md create mode 100644 doc/manual/src/protocols/tarball-fetcher.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 69c721b57..cfbb38be8 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -98,6 +98,8 @@ - [Channels](command-ref/files/channels.md) - [Default Nix expression](command-ref/files/default-nix-expression.md) - [Architecture](architecture/architecture.md) +- [Protocols](protocols/protocols.md) + - [Serving Tarball Flakes](protocols/tarball-fetcher.md) - [Glossary](glossary.md) - [Contributing](contributing/contributing.md) - [Hacking](contributing/hacking.md) diff --git a/doc/manual/src/protocols/protocols.md b/doc/manual/src/protocols/protocols.md new file mode 100644 index 000000000..d6bf1d809 --- /dev/null +++ b/doc/manual/src/protocols/protocols.md @@ -0,0 +1,4 @@ +# Protocols + +This chapter documents various developer-facing interfaces provided by +Nix. diff --git a/doc/manual/src/protocols/tarball-fetcher.md b/doc/manual/src/protocols/tarball-fetcher.md new file mode 100644 index 000000000..1c6c92000 --- /dev/null +++ b/doc/manual/src/protocols/tarball-fetcher.md @@ -0,0 +1,40 @@ +# Serving Tarball Flakes + +Tarball flakes are served as regular tarballs via HTTP or the file +system (for `file://` URLs). + +An HTTP server can return an "immutable" flakeref appropriate for lock +files. This allows users to specify a tarball flake input in +`flake.nix` that requests the latest version of a flake +(e.g. `https://example.org/hello/latest.tar.gz`), while `flake.lock` +will record a URL whose contents will not change +(e.g. `https://example.org/hello/.tar.gz`). To do so, the +server must return a `Link` header with the `rel` attribute set to +`immutable`, as follows: + +``` +Link: ; rel="immutable" +``` + +(Note the required `<` and `>` characters around *flakeref*.) + +*flakeref* must be a tarball flakeref. It can contain flake attributes +such as `narHash`, `rev` and `revCount`. If `narHash` is included, its +value must be the NAR hash of the unpacked tarball (as computed via +`nix hash path`). Nix checks the contents of the returned tarball +against the `narHash` attribute. The `rev` and `revCount` attributes +are useful when the tarball flake is a mirror of a fetcher type that +has those attributes, such as Git or GitHub. They are not checked by +Nix. + +``` +Link: ; rel="immutable" +``` + +(The linebreaks in this example are for clarity and must not be included in the actual response.) + +For tarball flakes, the value of the `lastModified` flake attribute is +defined as the timestamp of the newest file inside the tarball. From b1ed9b4b0cc037a8b14dcc37fd32f6a5a16e7ba3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 16 Jun 2023 16:48:37 +0200 Subject: [PATCH 36/48] Apply suggestions from code review Co-authored-by: Robert Hensing --- doc/manual/src/protocols/tarball-fetcher.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/doc/manual/src/protocols/tarball-fetcher.md b/doc/manual/src/protocols/tarball-fetcher.md index 1c6c92000..0d3212303 100644 --- a/doc/manual/src/protocols/tarball-fetcher.md +++ b/doc/manual/src/protocols/tarball-fetcher.md @@ -1,15 +1,17 @@ -# Serving Tarball Flakes +# Lockable HTTP Tarball Protocol -Tarball flakes are served as regular tarballs via HTTP or the file -system (for `file://` URLs). +Tarball flakes can be served as regular tarballs via HTTP or the file +system (for `file://` URLs). Unless the server implements the Lockable +HTTP Tarball protocol, it is the responsibility of the user to make sure that +the URL always produces the same tarball contents. -An HTTP server can return an "immutable" flakeref appropriate for lock +An HTTP server can return an "immutable" HTTP URL appropriate for lock files. This allows users to specify a tarball flake input in `flake.nix` that requests the latest version of a flake (e.g. `https://example.org/hello/latest.tar.gz`), while `flake.lock` will record a URL whose contents will not change (e.g. `https://example.org/hello/.tar.gz`). To do so, the -server must return a `Link` header with the `rel` attribute set to +server must return an [HTTP `Link` header](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Link) with the `rel` attribute set to `immutable`, as follows: ``` From 741f7837f85d1675efbcf43877bbcdfb9a862745 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christina=20S=C3=B8rensen?= <89321978+cafkafk@users.noreply.github.com> Date: Sat, 17 Jun 2023 09:06:17 +0000 Subject: [PATCH 37/48] Fix wikipedia links (#8533) --- doc/manual/src/architecture/architecture.md | 6 +++--- doc/manual/src/contributing/hacking.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index e51958052..9e969972e 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -7,11 +7,11 @@ It should help users understand why Nix behaves as it does, and it should help d Nix consists of [hierarchical layers]. -[hierarchical layers]: https://en.m.wikipedia.org/wiki/Multitier_architecture#Layers +[hierarchical layers]: https://en.wikipedia.org/wiki/Multitier_architecture#Layers The following [concept map] shows its main components (rectangles), the objects they operate on (rounded rectangles), and their interactions (connecting phrases): -[concept map]: https://en.m.wikipedia.org/wiki/Concept_map +[concept map]: https://en.wikipedia.org/wiki/Concept_map ``` @@ -76,7 +76,7 @@ The result of a build task can be input to another build task. The following [data flow diagram] shows a build plan for illustration. Build inputs used as instructions to a build task are marked accordingly: -[data flow diagram]: https://en.m.wikipedia.org/wiki/Data-flow_diagram +[data flow diagram]: https://en.wikipedia.org/wiki/Data-flow_diagram ``` +--------------------------------------------------------------------+ diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index b954a2167..c57d45138 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -378,7 +378,7 @@ rm $(git ls-files doc/manual/ -o | grep -F '.md') && rmdir doc/manual/src/comman [`mdbook-linkcheck`] does not implement checking [URI fragments] yet. [`mdbook-linkcheck`]: https://github.com/Michael-F-Bryan/mdbook-linkcheck -[URI fragments]: https://en.m.wikipedia.org/wiki/URI_fragment +[URI fragments]: https://en.wikipedia.org/wiki/URI_fragment #### `@docroot@` variable From b931d83550b200676c36ae8d1038a281ae4aa0ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Sat, 17 Jun 2023 15:05:10 +0200 Subject: [PATCH 38/48] ci: bump install-nix-action, don't fail fast --- .github/workflows/ci.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f1f6d43f..c3a17d106 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,6 +11,7 @@ jobs: tests: needs: [check_secrets] strategy: + fail-fast: false matrix: os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} @@ -19,7 +20,7 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v21 + - uses: cachix/install-nix-action@v22 with: # The sandbox would otherwise be disabled by default on Darwin extra_nix_config: "sandbox = true" @@ -61,7 +62,7 @@ jobs: with: fetch-depth: 0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v21 + - uses: cachix/install-nix-action@v22 with: install_url: https://releases.nixos.org/nix/nix-2.13.3/install - uses: cachix/cachix-action@v12 @@ -76,13 +77,14 @@ jobs: needs: [installer, check_secrets] if: github.event_name == 'push' && needs.check_secrets.outputs.cachix == 'true' strategy: + fail-fast: false matrix: os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v21 + - uses: cachix/install-nix-action@v22 with: install_url: '${{needs.installer.outputs.installerURL}}' install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" @@ -109,7 +111,7 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v21 + - uses: cachix/install-nix-action@v22 with: install_url: https://releases.nixos.org/nix/nix-2.13.3/install - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV From 6b06e97bde3d2d87a750210d46086624dc96a0f8 Mon Sep 17 00:00:00 2001 From: Adam Joseph Date: Thu, 15 Jun 2023 17:12:03 -0700 Subject: [PATCH 39/48] src/libexpr/eval.hh: add link for allowed-uris option This commit adds a link to the documentation for `--option allowed-uris` where that option is mentioned while describing `restrict-eval`. --- src/libexpr/eval.hh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index ea1f5975b..8e41bdbd0 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -741,7 +741,8 @@ struct EvalSettings : Config If set to `true`, the Nix evaluator will not allow access to any files outside of the Nix search path (as set via the `NIX_PATH` environment variable or the `-I` option), or to URIs outside of - `allowed-uris`. The default is `false`. + [`allowed-uris`](../command-ref/conf-file.md#conf-allowed-uris). + The default is `false`. )"}; Setting pureEval{this, false, "pure-eval", From 7bf17f8825b7aacde8b4e3c5e035f6d442d649c4 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 19 Jun 2023 05:45:08 +0200 Subject: [PATCH 40/48] Add description for file system objects (#8500) While this is not actually a notion in the implementation, it is explicitly described in the thesis and quite important for understanding how the store works. Co-authored-by: John Ericson Co-authored-by: Robert Hensing --- doc/manual/src/SUMMARY.md.in | 3 +- .../src/architecture/file-system-object.md | 64 +++++++++++++++++++ doc/manual/src/glossary.md | 15 +++-- 3 files changed, 76 insertions(+), 6 deletions(-) create mode 100644 doc/manual/src/architecture/file-system-object.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index cfbb38be8..13d2e4d15 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -97,7 +97,8 @@ - [manifest.json](command-ref/files/manifest.json.md) - [Channels](command-ref/files/channels.md) - [Default Nix expression](command-ref/files/default-nix-expression.md) -- [Architecture](architecture/architecture.md) +- [Architecture and Design](architecture/architecture.md) + - [File System Object](architecture/file-system-object.md) - [Protocols](protocols/protocols.md) - [Serving Tarball Flakes](protocols/tarball-fetcher.md) - [Glossary](glossary.md) diff --git a/doc/manual/src/architecture/file-system-object.md b/doc/manual/src/architecture/file-system-object.md new file mode 100644 index 000000000..42f047260 --- /dev/null +++ b/doc/manual/src/architecture/file-system-object.md @@ -0,0 +1,64 @@ +# File System Object + +Nix uses a simplified model of the file system, which consists of file system objects. +Every file system object is one of the following: + + - File + + - A possibly empty sequence of bytes for contents + - A single boolean representing the [executable](https://en.m.wikipedia.org/wiki/File-system_permissions#Permissions) permission + + - Directory + + Mapping of names to child file system objects + + - [Symbolic link](https://en.m.wikipedia.org/wiki/Symbolic_link) + + An arbitrary string. + Nix does not assign any semantics to symbolic links. + +File system objects and their children form a tree. +A bare file or symlink can be a root file system object. + +Nix does not encode any other file system notions such as [hard links](https://en.m.wikipedia.org/wiki/Hard_link), [permissions](https://en.m.wikipedia.org/wiki/File-system_permissions), timestamps, or other metadata. + +## Examples of file system objects + +A plain file: + +``` +50 B, executable: false +``` + +An executable file: + +``` +122 KB, executable: true +``` + +A symlink: + +``` +-> /usr/bin/sh +``` + +A directory with contents: + +``` +├── bin +│   └── hello: 35 KB, executable: true +└── share + ├── info + │   └── hello.info: 36 KB, executable: false + └── man + └── man1 + └── hello.1.gz: 790 B, executable: false +``` + +A directory that contains a symlink and other directories: + +``` +├── bin -> share/go/bin +├── nix-support/ +└── share/ +``` diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 47a484826..ac0bb3c2f 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -85,12 +85,17 @@ [store path]: #gloss-store-path + - [file system object]{#gloss-store-object}\ + The Nix data model for representing simplified file system data. + + See [File System Object](@docroot@/architecture/file-system-object.md) for details. + + [file system object]: #gloss-file-system-object + - [store object]{#gloss-store-object}\ - A file that is an immediate child of the Nix store directory. These - can be regular files, but also entire directory trees. Store objects - can be sources (objects copied from outside of the store), - derivation outputs (objects produced by running a build task), or - derivations (files describing a build task). + + A store object consists of a [file system object], [reference]s to other store objects, and other metadata. + It can be referred to by a [store path]. [store object]: #gloss-store-object From c404623a1d39431cf7b4ccd0b0b396a821a6eade Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 19 Jun 2023 00:04:59 -0400 Subject: [PATCH 41/48] Clean up a few things related to profiles (#8526) - Greatly expand API docs - Clean up code in misc ways - Instead of a complicated single loop on generations, do different operations in successive subsequent steps. - Avoid `ref` in one place where `&` is fine - Just return path instead of mutating an argument in `makeName` Co-authored-by: Valentin Gagarin --- src/libcmd/command.cc | 4 +- src/libstore/profiles.cc | 123 +++++++++------ src/libstore/profiles.hh | 143 +++++++++++++++++- .../nix-collect-garbage.cc | 7 +- src/nix-env/nix-env.cc | 7 +- src/nix-env/user-env.cc | 2 +- src/nix/profile.cc | 7 +- 7 files changed, 224 insertions(+), 69 deletions(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 6c4648b34..4fc197956 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -239,9 +239,7 @@ void MixProfile::updateProfile(const StorePath & storePath) if (!store) throw Error("'--profile' is not supported for this Nix store"); auto profile2 = absPath(*profile); switchLink(profile2, - createGeneration( - ref(store), - profile2, storePath)); + createGeneration(*store, profile2, storePath)); } void MixProfile::updateProfile(const BuiltPaths & buildables) diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index ba5c8583f..4e9955948 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -13,8 +13,10 @@ namespace nix { -/* Parse a generation name of the format - `--link'. */ +/** + * Parse a generation name of the format + * `--link'. + */ static std::optional parseName(const std::string & profileName, const std::string & name) { if (name.substr(0, profileName.size() + 1) != profileName + "-") return {}; @@ -28,7 +30,6 @@ static std::optional parseName(const std::string & profileName } - std::pair> findGenerations(Path profile) { Generations gens; @@ -61,15 +62,16 @@ std::pair> findGenerations(Path pro } -static void makeName(const Path & profile, GenerationNumber num, - Path & outLink) +/** + * Create a generation name that can be parsed by `parseName()`. + */ +static Path makeName(const Path & profile, GenerationNumber num) { - Path prefix = fmt("%1%-%2%", profile, num); - outLink = prefix + "-link"; + return fmt("%s-%s-link", profile, num); } -Path createGeneration(ref store, Path profile, StorePath outPath) +Path createGeneration(LocalFSStore & store, Path profile, StorePath outPath) { /* The new generation number should be higher than old the previous ones. */ @@ -79,7 +81,7 @@ Path createGeneration(ref store, Path profile, StorePath outPath) if (gens.size() > 0) { Generation last = gens.back(); - if (readLink(last.path) == store->printStorePath(outPath)) { + if (readLink(last.path) == store.printStorePath(outPath)) { /* We only create a new generation symlink if it differs from the last one. @@ -89,7 +91,7 @@ Path createGeneration(ref store, Path profile, StorePath outPath) return last.path; } - num = gens.back().number; + num = last.number; } else { num = 0; } @@ -100,9 +102,8 @@ Path createGeneration(ref store, Path profile, StorePath outPath) to the permanent roots (of which the GC would have a stale view). If we didn't do it this way, the GC might remove the user environment etc. we've just built. */ - Path generation; - makeName(profile, num + 1, generation); - store->addPermRoot(outPath, generation); + Path generation = makeName(profile, num + 1); + store.addPermRoot(outPath, generation); return generation; } @@ -117,12 +118,19 @@ static void removeFile(const Path & path) void deleteGeneration(const Path & profile, GenerationNumber gen) { - Path generation; - makeName(profile, gen, generation); + Path generation = makeName(profile, gen); removeFile(generation); } - +/** + * Delete a generation with dry-run mode. + * + * Like `deleteGeneration()` but: + * + * - We log what we are going to do. + * + * - We only actually delete if `dryRun` is false. + */ static void deleteGeneration2(const Path & profile, GenerationNumber gen, bool dryRun) { if (dryRun) @@ -150,27 +158,36 @@ void deleteGenerations(const Path & profile, const std::set & } } +/** + * Advanced the iterator until the given predicate `cond` returns `true`. + */ +static inline void iterDropUntil(Generations & gens, auto && i, auto && cond) +{ + for (; i != gens.rend() && !cond(*i); ++i); +} + void deleteGenerationsGreaterThan(const Path & profile, GenerationNumber max, bool dryRun) { + if (max == 0) + throw Error("Must keep at least one generation, otherwise the current one would be deleted"); + PathLocks lock; lockProfile(lock, profile); - bool fromCurGen = false; - auto [gens, curGen] = findGenerations(profile); - for (auto i = gens.rbegin(); i != gens.rend(); ++i) { - if (i->number == curGen) { - fromCurGen = true; - max--; - continue; - } - if (fromCurGen) { - if (max) { - max--; - continue; - } - deleteGeneration2(profile, i->number, dryRun); - } - } + auto [gens, _curGen] = findGenerations(profile); + auto curGen = _curGen; + + auto i = gens.rbegin(); + + // Find the current generation + iterDropUntil(gens, i, [&](auto & g) { return g.number == curGen; }); + + // Skip over `max` generations, preserving them + for (auto keep = 0; i != gens.rend() && keep < max; ++i, ++keep); + + // Delete the rest + for (; i != gens.rend(); ++i) + deleteGeneration2(profile, i->number, dryRun); } void deleteOldGenerations(const Path & profile, bool dryRun) @@ -193,23 +210,33 @@ void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun) auto [gens, curGen] = findGenerations(profile); - bool canDelete = false; - for (auto i = gens.rbegin(); i != gens.rend(); ++i) - if (canDelete) { - assert(i->creationTime < t); - if (i->number != curGen) - deleteGeneration2(profile, i->number, dryRun); - } else if (i->creationTime < t) { - /* We may now start deleting generations, but we don't - delete this generation yet, because this generation was - still the one that was active at the requested point in - time. */ - canDelete = true; - } + auto i = gens.rbegin(); + + // Predicate that the generation is older than the given time. + auto older = [&](auto & g) { return g.creationTime < t; }; + + // Find the first older generation, if one exists + iterDropUntil(gens, i, older); + + /* Take the previous generation + + We don't want delete this one yet because it + existed at the requested point in time, and + we want to be able to roll back to it. */ + if (i != gens.rend()) ++i; + + // Delete all previous generations (unless current). + for (; i != gens.rend(); ++i) { + /* Creating date and generations should be monotonic, so lower + numbered derivations should also be older. */ + assert(older(*i)); + if (i->number != curGen) + deleteGeneration2(profile, i->number, dryRun); + } } -void deleteGenerationsOlderThan(const Path & profile, std::string_view timeSpec, bool dryRun) +time_t parseOlderThanTimeSpec(std::string_view timeSpec) { if (timeSpec.empty() || timeSpec[timeSpec.size() - 1] != 'd') throw UsageError("invalid number of days specifier '%1%', expected something like '14d'", timeSpec); @@ -221,9 +248,7 @@ void deleteGenerationsOlderThan(const Path & profile, std::string_view timeSpec, if (!days || *days < 1) throw UsageError("invalid number of days specifier '%1%'", timeSpec); - time_t oldTime = curTime - *days * 24 * 3600; - - deleteGenerationsOlderThan(profile, oldTime, dryRun); + return curTime - *days * 24 * 3600; } diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh index 4e1f42e83..193c0bf21 100644 --- a/src/libstore/profiles.hh +++ b/src/libstore/profiles.hh @@ -1,7 +1,11 @@ #pragma once -///@file +/** + * @file Implementation of Profiles. + * + * See the manual for additional information. + */ - #include "types.hh" +#include "types.hh" #include "pathlocks.hh" #include @@ -12,41 +16,166 @@ namespace nix { class StorePath; +/** + * A positive number identifying a generation for a given profile. + * + * Generation numbers are assigned sequentially. Each new generation is + * assigned 1 + the current highest generation number. + */ typedef uint64_t GenerationNumber; +/** + * A generation is a revision of a profile. + * + * Each generation is a mapping (key-value pair) from an identifier + * (`number`) to a store object (specified by `path`). + */ struct Generation { + /** + * The number of a generation is its unique identifier within the + * profile. + */ GenerationNumber number; + /** + * The store path identifies the store object that is the contents + * of the generation. + * + * These store paths / objects are not unique to the generation + * within a profile. Nix tries to ensure successive generations have + * distinct contents to avoid bloat, but nothing stops two + * non-adjacent generations from having the same contents. + * + * @todo Use `StorePath` instead of `Path`? + */ Path path; + + /** + * When the generation was created. This is extra metadata about the + * generation used to make garbage collecting old generations more + * convenient. + */ time_t creationTime; }; +/** + * All the generations of a profile + */ typedef std::list Generations; /** - * Returns the list of currently present generations for the specified - * profile, sorted by generation number. Also returns the number of - * the current generation. + * Find all generations for the given profile. + * + * @param profile A profile specified by its name and location combined + * into a path. E.g. if "foo" is the name of the profile, and "/bar/baz" + * is the directory it is in, then the path "/bar/baz/foo" would be the + * argument for this parameter. + * + * @return The pair of: + * + * - The list of currently present generations for the specified profile, + * sorted by ascending generation number. + * + * - The number of the current/active generation. + * + * Note that the current/active generation need not be the latest one. */ std::pair> findGenerations(Path profile); class LocalFSStore; -Path createGeneration(ref store, Path profile, StorePath outPath); +/** + * Create a new generation of the given profile + * + * If the previous generation (not the currently active one!) has a + * distinct store object, a fresh generation number is mapped to the + * given store object, referenced by path. Otherwise, the previous + * generation is assumed. + * + * The behavior of reusing existing generations like this makes this + * procedure idempotent. It also avoids clutter. + */ +Path createGeneration(LocalFSStore & store, Path profile, StorePath outPath); +/** + * Unconditionally delete a generation + * + * @param profile A profile specified by its name and location combined into a path. + * + * @param gen The generation number specifying exactly which generation + * to delete. + * + * Because there is no check of whether the generation to delete is + * active, this is somewhat unsafe. + * + * @todo Should we expose this at all? + */ void deleteGeneration(const Path & profile, GenerationNumber gen); +/** + * Delete the given set of generations. + * + * @param profile The profile, specified by its name and location combined into a path, whose generations we want to delete. + * + * @param gensToDelete The generations to delete, specified by a set of + * numbers. + * + * @param dryRun Log what would be deleted instead of actually doing + * so. + * + * Trying to delete the currently active generation will fail, and cause + * no generations to be deleted. + */ void deleteGenerations(const Path & profile, const std::set & gensToDelete, bool dryRun); +/** + * Delete generations older than `max` passed the current generation. + * + * @param profile The profile, specified by its name and location combined into a path, whose generations we want to delete. + * + * @param max How many generations to keep up to the current one. Must + * be at least 1 so we don't delete the current one. + * + * @param dryRun Log what would be deleted instead of actually doing + * so. + */ void deleteGenerationsGreaterThan(const Path & profile, GenerationNumber max, bool dryRun); +/** + * Delete all generations other than the current one + * + * @param profile The profile, specified by its name and location combined into a path, whose generations we want to delete. + * + * @param dryRun Log what would be deleted instead of actually doing + * so. + */ void deleteOldGenerations(const Path & profile, bool dryRun); +/** + * Delete generations older than `t`, except for the most recent one + * older than `t`. + * + * @param profile The profile, specified by its name and location combined into a path, whose generations we want to delete. + * + * @param dryRun Log what would be deleted instead of actually doing + * so. + */ void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun); -void deleteGenerationsOlderThan(const Path & profile, std::string_view timeSpec, bool dryRun); +/** + * Parse a temp spec intended for `deleteGenerationsOlderThan()`. + * + * Throws an exception if `timeSpec` fails to parse. + */ +time_t parseOlderThanTimeSpec(std::string_view timeSpec); +/** + * Smaller wrapper around `replaceSymlink` for replacing the current + * generation of a profile. Does not enforce proper structure. + * + * @todo Always use `switchGeneration()` instead, and delete this. + */ void switchLink(Path link, Path target); /** diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index cb1f42e35..70af53b28 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -41,9 +41,10 @@ void removeOldGenerations(std::string dir) } if (link.find("link") != std::string::npos) { printInfo("removing old generations of profile %s", path); - if (deleteOlderThan != "") - deleteGenerationsOlderThan(path, deleteOlderThan, dryRun); - else + if (deleteOlderThan != "") { + auto t = parseOlderThanTimeSpec(deleteOlderThan); + deleteGenerationsOlderThan(path, t, dryRun); + } else deleteOldGenerations(path, dryRun); } } else if (type == DT_DIR) { diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 5e94f2d14..91b073b49 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -772,7 +772,7 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) debug("switching to new user environment"); Path generation = createGeneration( - ref(store2), + *store2, globals.profile, drv.queryOutPath()); switchLink(globals.profile, generation); @@ -1356,13 +1356,14 @@ static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opAr if (opArgs.size() == 1 && opArgs.front() == "old") { deleteOldGenerations(globals.profile, globals.dryRun); } else if (opArgs.size() == 1 && opArgs.front().find('d') != std::string::npos) { - deleteGenerationsOlderThan(globals.profile, opArgs.front(), globals.dryRun); + auto t = parseOlderThanTimeSpec(opArgs.front()); + deleteGenerationsOlderThan(globals.profile, t, globals.dryRun); } else if (opArgs.size() == 1 && opArgs.front().find('+') != std::string::npos) { if (opArgs.front().size() < 2) throw Error("invalid number of generations '%1%'", opArgs.front()); auto str_max = opArgs.front().substr(1); auto max = string2Int(str_max); - if (!max || *max == 0) + if (!max) throw Error("invalid number of generations to keep '%1%'", opArgs.front()); deleteGenerationsGreaterThan(globals.profile, *max, globals.dryRun); } else { diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 9e916abc4..d12d70f33 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -158,7 +158,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, } debug("switching to new user environment"); - Path generation = createGeneration(ref(store2), profile, topLevelOut); + Path generation = createGeneration(*store2, profile, topLevelOut); switchLink(profile, generation); } diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 7cea616d2..f3b73f10d 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -806,9 +806,10 @@ struct CmdProfileWipeHistory : virtual StoreCommand, MixDefaultProfile, MixDryRu void run(ref store) override { - if (minAge) - deleteGenerationsOlderThan(*profile, *minAge, dryRun); - else + if (minAge) { + auto t = parseOlderThanTimeSpec(*minAge); + deleteGenerationsOlderThan(*profile, t, dryRun); + } else deleteOldGenerations(*profile, dryRun); } }; From 966e5dc991e849d2cd26d19266ae3f065ec69c96 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 19 Jun 2023 10:39:19 +0200 Subject: [PATCH 42/48] CONTRIBUTING.md: add link to "good first issues" --- CONTRIBUTING.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 57a949906..4a72a8eac 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,7 +5,6 @@ We appreciate your support. Reading and following these guidelines will help us make the contribution process easy and effective for everyone involved. - ## Report a bug 1. Check on the [GitHub issue tracker](https://github.com/NixOS/nix/issues) if your bug was already reported. @@ -30,6 +29,8 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). You can use [labels](https://github.com/NixOS/nix/labels) to filter for relevant topics. 2. Search for related issues that cover what you're going to work on. It could help to mention there that you will work on the issue. + + Issues labeled ["good first issue"](https://github.com/NixOS/nix/labels/good-first-issue) should be relatively easy to fix and are likely to get merged quickly. Pull requests addressing issues labeled ["idea approved"](https://github.com/NixOS/nix/labels/idea%20approved) are especially welcomed by maintainers and will receive prioritised review. 3. Check the [Nix reference manual](https://nixos.org/manual/nix/unstable/contributing/hacking.html) for information on building Nix and running its tests. From b6e74ea5a82e5c43d42bf8386236947408748a96 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 19 Jun 2023 10:55:34 +0200 Subject: [PATCH 43/48] maintainers: add note on marking PRs as draft as discussed with maintainers team --- maintainers/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/maintainers/README.md b/maintainers/README.md index d13349438..0d520cb0c 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -117,6 +117,7 @@ Pull requests in this column are reviewed together during work meetings. This is both for spreading implementation knowledge and for establishing common values in code reviews. When the overall direction is agreed upon, even when further changes are required, the pull request is assigned to one team member. +If significant changes are requested or reviewers cannot come to a conclusion in reasonable time, the pull request is [marked as draft](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/changing-the-stage-of-a-pull-request#converting-a-pull-request-to-a-draft). ### Assigned From feb8d552aeca36adde5a9c25b92fd9eb205a761e Mon Sep 17 00:00:00 2001 From: Ben Radford <104896700+benradf@users.noreply.github.com> Date: Mon, 19 Jun 2023 13:22:41 +0100 Subject: [PATCH 44/48] Update src/libstore/local-store.hh Co-authored-by: Valentin Gagarin --- src/libstore/local-store.hh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 0e7b1334f..bd4794eb8 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -52,8 +52,7 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig R"( Allow this store to be opened when its [database](@docroot@/glossary.md#gloss-nix-database) is on a read-only filesystem. - Normally Nix will attempt to open the store database in read-write mode, even for querying (when write access is not needed). - This causes it to fail if the database is on a read-only filesystem. + Normally Nix will attempt to open the store database in read-write mode, even for querying (when write access is not needed), causing it to fail if the database is on a read-only filesystem. Enable read-only mode to disable locking and open the SQLite database with the [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set. From ef40448b1cb33165af5522f72d5d352d66035671 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Mon, 19 Jun 2023 13:47:21 +0100 Subject: [PATCH 45/48] Remove redundant description on experimental flag. --- src/libstore/local-store.hh | 1 + src/libutil/experimental-features.cc | 8 -------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index bd4794eb8..1af64c77b 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -59,6 +59,7 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig **Warning** Do not use this unless the filesystem is read-only. Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process. + While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it. )"}; const std::string name() override { return "Local Store"; } diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 7c2b29a07..3a04083ed 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -226,14 +226,6 @@ constexpr std::array xpFeatureDetails = {{ .name = "read-only-local-store", .description = R"( Allow the use of the `read-only` parameter in local store URIs. - - Set this parameter to `true` to allow stores with databases on read-only filesystems to be opened for querying; ordinarily Nix will refuse to do this. - - This is because SQLite requires write access to the database file to perform the file locking operations necessary for safe concurrent access. - When `read-only` is set to `true`, the database will be opened in immutable mode. - - Under this mode, SQLite will not do any locking at all, so you should be certain that the database will not be changed. - While the filesystem the database resides on might be read-only to this process, consider whether another user, process, or system, might have write access to it. )", }, }}; From b09baa3bc31226cac5ccdbd7ba9d72a2ed389207 Mon Sep 17 00:00:00 2001 From: Ben Radford Date: Mon, 19 Jun 2023 13:57:10 +0100 Subject: [PATCH 46/48] Link to LocalStore section of nix3-help-stores section. --- src/libutil/experimental-features.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 3a04083ed..7c4112d32 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -225,7 +225,7 @@ constexpr std::array xpFeatureDetails = {{ .tag = Xp::ReadOnlyLocalStore, .name = "read-only-local-store", .description = R"( - Allow the use of the `read-only` parameter in local store URIs. + Allow the use of the `read-only` parameter in [local store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) URIs. )", }, }}; From ba492a98baa0fa37b0914e3f7d00f2859cef5bcd Mon Sep 17 00:00:00 2001 From: Ben Radford <104896700+benradf@users.noreply.github.com> Date: Mon, 19 Jun 2023 14:07:31 +0100 Subject: [PATCH 47/48] Update src/libstore/local-store.hh Co-authored-by: Valentin Gagarin --- src/libstore/local-store.hh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 1af64c77b..8a3b0b43f 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -56,10 +56,11 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig Enable read-only mode to disable locking and open the SQLite database with the [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set. - **Warning** - Do not use this unless the filesystem is read-only. - Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process. - While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it. + > **Warning** + > Do not use this unless the filesystem is read-only. + > + > Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process. + > While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it. )"}; const std::string name() override { return "Local Store"; } From 4e72b8483ede9242ddde99eb8011b18098172444 Mon Sep 17 00:00:00 2001 From: Ben Radford <104896700+benradf@users.noreply.github.com> Date: Mon, 19 Jun 2023 16:01:43 +0100 Subject: [PATCH 48/48] Update src/libstore/sqlite.hh Co-authored-by: John Ericson --- src/libstore/sqlite.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index b0e84f8ed..0c08267f7 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -29,7 +29,7 @@ enum class SQLiteOpenMode { * Use this mode if the database is on a read-only filesystem. * Fails with an error if the database does not exist. */ - Immutable + Immutable, }; /**