From 408dcfc0d3898fb855c1038d24c09c82fe58c379 Mon Sep 17 00:00:00 2001 From: Sandro Date: Mon, 5 Sep 2022 15:42:10 +0200 Subject: [PATCH 001/139] Improve experimental-features error wording --- src/libutil/experimental-features.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index fa79cca6b..d8b0d30fa 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -52,7 +52,7 @@ std::set parseFeatures(const std::set & rawFea } MissingExperimentalFeature::MissingExperimentalFeature(ExperimentalFeature feature) - : Error("experimental Nix feature '%1%' is disabled; use '--extra-experimental-features %1%' to override", showExperimentalFeature(feature)) + : Error("experimental Nix feature '%1%' is disabled; pass '--extra-experimental-features %1%' as an argument to enable it", showExperimentalFeature(feature)) , missingFeature(feature) {} From 481e4082bf96f5613693785e97a9a5cec7c09940 Mon Sep 17 00:00:00 2001 From: Sandro Date: Wed, 9 Nov 2022 12:03:53 +0100 Subject: [PATCH 002/139] Update src/libutil/experimental-features.cc Co-authored-by: Valentin Gagarin --- src/libutil/experimental-features.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index d8b0d30fa..052988b34 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -52,7 +52,7 @@ std::set parseFeatures(const std::set & rawFea } MissingExperimentalFeature::MissingExperimentalFeature(ExperimentalFeature feature) - : Error("experimental Nix feature '%1%' is disabled; pass '--extra-experimental-features %1%' as an argument to enable it", showExperimentalFeature(feature)) + : Error("experimental Nix feature '%1%' is disabled; add '--extra-experimental-features %1%' to enable it", showExperimentalFeature(feature)) , missingFeature(feature) {} From c74248c56e6e5558603be6c6eda43eb439772f20 Mon Sep 17 00:00:00 2001 From: Michael Lohmann Date: Thu, 12 Jan 2023 15:45:19 +0100 Subject: [PATCH 003/139] install-multi-user: ignore profile_target backups that have no change If there was a prior nix installation that created this backup file and then you tried to install it again, it would stop to tell you there is this file. But if the file and its backup are identical in content, there is no harm in continuing and in a later step overwriting the existing backup file with the identical one. This is just a convenience feature. --- scripts/install-multi-user.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index f149ea0d7..b2f2a9e45 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -445,6 +445,14 @@ EOF # a row for different files. if [ -e "$profile_target$PROFILE_BACKUP_SUFFIX" ]; then # this backup process first released in Nix 2.1 + + if diff -q "$profile_target$PROFILE_BACKUP_SUFFIX" "$profile_target" > /dev/null; then + # a backup file for the rc-file exist, but they are identical, + # so we can safely ignore it and overwrite it with the same + # content later + continue + fi + failure < Date: Mon, 23 Jan 2023 10:50:44 +0000 Subject: [PATCH 004/139] improved help command listing. --- src/libcmd/repl.cc | 53 +++++++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index e3afb1531..5fc503c2b 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -486,35 +486,40 @@ bool NixRepl::processLine(std::string line) std::cout << "The following commands are available:\n" << "\n" - << " Evaluate and print expression\n" - << " = Bind expression to variable\n" - << " :a Add attributes from resulting set to scope\n" - << " :b Build a derivation\n" - << " :bl Build a derivation, creating GC roots in the working directory\n" - << " :e Open package or function in $EDITOR\n" - << " :i Build derivation, then install result into current profile\n" - << " :l Load Nix expression and add it to scope\n" - << " :lf Load Nix flake and add it to scope\n" - << " :p Evaluate and print expression recursively\n" - << " :q Exit nix-repl\n" - << " :r Reload all files\n" - << " :sh Build dependencies of derivation, then start nix-shell\n" - << " :t Describe result of evaluation\n" - << " :u Build derivation, then start nix-shell\n" - << " :doc Show documentation of a builtin function\n" - << " :log Show logs for a derivation\n" - << " :te [bool] Enable, disable or toggle showing traces for errors\n" + << " Evaluate and print expression\n" + << " = Bind expression to variable\n" + << " :a, :add Add attributes from resulting set to scope\n" + << " :b Build a derivation\n" + << " :bl Build a derivation, creating GC roots in the\n" + << " working directory\n" + << " :e, :edit Open package or function in $EDITOR\n" + << " :i Build derivation, then install result into\n" + << " current profile\n" + << " :l, :load Load Nix expression and add it to scope\n" + << " :lf, :load-flake Load Nix flake and add it to scope\n" + << " :p, :print Evaluate and print expression recursively\n" + << " :q, :quit Exit nix-repl\n" + << " :r, :reload Reload all files\n" + << " :sh Build dependencies of derivation, then start\n" + << " nix-shell\n" + << " :t Describe result of evaluation\n" + << " :u Build derivation, then start nix-shell\n" + << " :doc Show documentation of a builtin function\n" + << " :log Show logs for a derivation\n" + << " :te, :trace-enable [bool] Enable, disable or toggle showing traces for\n" + << " errors\n" + << " :?, :help Brings up this help menu\n" ; if (state->debugRepl) { std::cout << "\n" << " Debug mode commands\n" - << " :env Show env stack\n" - << " :bt Show trace stack\n" - << " :st Show current trace\n" - << " :st Change to another trace in the stack\n" - << " :c Go until end of program, exception, or builtins.break\n" - << " :s Go one step\n" + << " :env Show env stack\n" + << " :bt, :backtrace Show trace stack\n" + << " :st Show current trace\n" + << " :st Change to another trace in the stack\n" + << " :c, :continue Go until end of program, exception, or builtins.break\n" + << " :s, :step Go one step\n" ; } From c4df53f1542086836d6d59cf82cff8f5cbe5d8f1 Mon Sep 17 00:00:00 2001 From: Rynn Blackmon Date: Wed, 19 Apr 2023 16:32:47 -0700 Subject: [PATCH 005/139] Mention `$DRV_PATH` in post-build-hook docs --- doc/manual/src/advanced-topics/post-build-hook.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/manual/src/advanced-topics/post-build-hook.md b/doc/manual/src/advanced-topics/post-build-hook.md index 1479cc3a4..17b67111c 100644 --- a/doc/manual/src/advanced-topics/post-build-hook.md +++ b/doc/manual/src/advanced-topics/post-build-hook.md @@ -69,6 +69,8 @@ exec nix copy --to "s3://example-nix-cache" $OUT_PATHS > store sign`. Nix guarantees the paths will not contain any spaces, > however a store path might contain glob characters. The `set -f` > disables globbing in the shell. +> If you want to upload the .drv file too, the `$DRV_PATH` variable +> is also defined for the script and works just like `$OUT_PATHS`. Then make sure the hook program is executable by the `root` user: From 3b3822ea1ddf34f8fcb44c67cda2b778b5b41a34 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 7 Jul 2023 15:08:25 +0200 Subject: [PATCH 006/139] tests: Reformat exit code error message Now looks like: Expected exit code '123' but got '0' from command 'echo' 'hi' --- tests/common/vars-and-functions.sh.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/common/vars-and-functions.sh.in b/tests/common/vars-and-functions.sh.in index a9e6c802f..33a764c3f 100644 --- a/tests/common/vars-and-functions.sh.in +++ b/tests/common/vars-and-functions.sh.in @@ -195,7 +195,7 @@ expect() { shift "$@" && res=0 || res="$?" if [[ $res -ne $expected ]]; then - echo "Expected '$expected' but got '$res' while running '${*@Q}'" >&2 + echo "Expected exit code '$expected' but got '$res' from command ${*@Q}" >&2 return 1 fi return 0 @@ -209,7 +209,7 @@ expectStderr() { shift "$@" 2>&1 && res=0 || res="$?" if [[ $res -ne $expected ]]; then - echo "Expected '$expected' but got '$res' while running '${*@Q}'" >&2 + echo "Expected exit code '$expected' but got '$res' from command ${*@Q}" >&2 return 1 fi return 0 From feb01b22ed2954232455b5c8346cc5ed09c21f81 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 17 Jul 2023 11:28:12 +0200 Subject: [PATCH 007/139] add links to store API documentation --- src/libstore/store-api.hh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 14a862eef..43aa00637 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -136,19 +136,22 @@ struct StoreConfig : public Config Setting priority{this, 0, "priority", R"( - Priority of this store when used as a substituter. A lower value means a higher priority. + Priority of this store when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). + A lower value means a higher priority. )"}; Setting wantMassQuery{this, false, "want-mass-query", R"( - Whether this store (when used as a substituter) can be - queried efficiently for path validity. + Whether this store can be queried efficiently for path validity when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). )"}; Setting systemFeatures{this, getDefaultSystemFeatures(), "system-features", - "Optional features that the system this store builds on implements (like \"kvm\")."}; + R"( + Optional [system features](@docroot@/command-ref/conf-file.md#conf-system-features) available on the system this store uses to build derivations. + Example: `"kvm"` + )" }; }; class Store : public std::enable_shared_from_this, public virtual StoreConfig From 493ddf617f0b2c406dc35026b6d481a69cf1d934 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 17 Jul 2023 11:28:27 +0200 Subject: [PATCH 008/139] reformat `system-features` setting documentation --- src/libstore/globals.hh | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index d4b8fb1f9..879588375 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -689,19 +689,21 @@ public: getDefaultSystemFeatures(), "system-features", R"( - A set of system “features” supported by this machine, e.g. `kvm`. - Derivations can express a dependency on such features through the - derivation attribute `requiredSystemFeatures`. For example, the - attribute + A set of system “features” supported by this machine. + Derivations can express a dependency on such features through the derivation attribute `requiredSystemFeatures`. + + For example, the attribute requiredSystemFeatures = [ "kvm" ]; - ensures that the derivation can only be built on a machine with the - `kvm` feature. + ensures that the derivation can only be built on a machine with the `kvm` feature. - This setting by default includes `kvm` if `/dev/kvm` is accessible, - and the pseudo-features `nixos-test`, `benchmark` and `big-parallel` - that are used in Nixpkgs to route builds to specific machines. + This setting by default includes + - `kvm` if `/dev/kvm` is accessible + - historical pseudo-features for backwards compatibility, used in Nixpkgs to route Hydra builds to specific machines + - `nixos-test` + - `benchmark` + - `big-parallel` )", {}, false}; Setting substituters{ From 5f37ebcf83c856b08a63778e07bfc5c84ea4a5ec Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 19 Jul 2023 11:57:20 +0200 Subject: [PATCH 009/139] document all special system features and their behavior --- .../src/language/advanced-attributes.md | 12 ++++++ doc/manual/src/release-notes/rl-2.12.md | 14 +------ src/libstore/globals.hh | 39 +++++++++++++------ 3 files changed, 41 insertions(+), 24 deletions(-) diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 307971434..2961f9dcc 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -345,3 +345,15 @@ Derivations can declare some infrequently used optional attributes. This is useful, for example, when generating self-contained filesystem images with their own embedded Nix store: hashes found inside such an image refer to the embedded store and not to the host's Nix store. + +- [`requiredSystemFeatures`]{#adv-attr-requiredSystemFeatures}\ + + If a derivation has the `requiredSystemFeatures` attribute, then Nix will only build it on a machine that has the corresponding features set in its [`system-features` configuration](@docroot@/command-ref/conf-file.md#conf-system-features). + + For example, setting + + ```nix + requiredSystemFeatures = [ "kvm" ]; + ``` + + ensures that the derivation can only be built on a machine with the `kvm` feature. diff --git a/doc/manual/src/release-notes/rl-2.12.md b/doc/manual/src/release-notes/rl-2.12.md index e2045d7bf..57d092e01 100644 --- a/doc/manual/src/release-notes/rl-2.12.md +++ b/doc/manual/src/release-notes/rl-2.12.md @@ -2,20 +2,8 @@ * On Linux, Nix can now run builds in a user namespace where they run as root (UID 0) and have 65,536 UIDs available. - - This is primarily useful for running containers such as `systemd-nspawn` - inside a Nix build. For an example, see [`tests/systemd-nspawn/nix`][nspawn]. - [nspawn]: https://github.com/NixOS/nix/blob/67bcb99700a0da1395fa063d7c6586740b304598/tests/systemd-nspawn.nix. - - A build can enable this by setting the derivation attribute: - - ``` - requiredSystemFeatures = [ "uid-range" ]; - ``` - - The `uid-range` [system feature] requires the [`auto-allocate-uids`] - setting to be enabled. + This can be used by requiring `uid-range` [system feature] in derivations. [system feature]: ../command-ref/conf-file.md#conf-system-features diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 879588375..9f852dc7b 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -337,7 +337,7 @@ public: users in `build-users-group`. UIDs are allocated starting at 872415232 (0x34000000) on Linux and 56930 on macOS. - )"}; + )", {}, true, Xp::AutoAllocateUids}; Setting startId{this, #if __linux__ @@ -690,20 +690,37 @@ public: "system-features", R"( A set of system “features” supported by this machine. - Derivations can express a dependency on such features through the derivation attribute `requiredSystemFeatures`. - For example, the attribute + This complements the [`system`](#conf-system) and [`extra-platforms`](#conf-extra-platforms) configuration options and the corresponding [`system`](@docroot@/language/derivations.md#attr-system) attribute on derivations. - requiredSystemFeatures = [ "kvm" ]; + Derivations can require system features in the derivation attribute [`requiredSystemFeatures`](@docroot@/language/advanced-attributes.md#adv-attr-requiredSystemFeatures). - ensures that the derivation can only be built on a machine with the `kvm` feature. + System features are generally user-defined, but the following have special treatment: - This setting by default includes - - `kvm` if `/dev/kvm` is accessible - - historical pseudo-features for backwards compatibility, used in Nixpkgs to route Hydra builds to specific machines - - `nixos-test` - - `benchmark` - - `big-parallel` + - `kvm` + + Set by default if `/dev/kvm` is accessible. + + - `nixos-test`, `benchmark`, `big-parallel` + + These historical pseudo-features are always enabled for backwards compatibility, used in Nixpkgs to route Hydra builds to specific machines. + + - `ca-derivations` + + Set by default if the [`ca-derivations` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) is enabled. + + - `recursive-nix` + + Set by default if the [`recursive-nix` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-recursive-nix) is enabled. + + - `uid-range` + + On Linux, Nix can run builds in a user namespace where they run as root (UID 0) and have 65,536 UIDs available. + This is primarily useful for running containers such as `systemd-nspawn` inside a Nix build. For an example, see [`tests/systemd-nspawn/nix`][nspawn]. + + [nspawn]: https://github.com/NixOS/nix/blob/67bcb99700a0da1395fa063d7c6586740b304598/tests/systemd-nspawn.nix. + + Set by default on Linux if the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting is enabled. )", {}, false}; Setting substituters{ From 2fa90e5824635998b445b6862bb49469e8fc75de Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 19 Jul 2023 13:59:18 +0200 Subject: [PATCH 010/139] add more details on CA derivations --- doc/manual/src/language/advanced-attributes.md | 2 ++ src/libstore/globals.hh | 16 +++++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 2961f9dcc..5292a2ffc 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -229,6 +229,8 @@ Derivations can declare some infrequently used optional attributes. [`outputHashAlgo`](#adv-attr-outputHashAlgo) like for *fixed-output derivations* (see above). + It also implicitly requires that the machine to build the derivation must have the `ca-derivations` [system feature](@docroot@/command-ref/conf-file.md#conf-system-features). + - [`passAsFile`]{#adv-attr-passAsFile}\ A list of names of attributes that should be passed via files rather than environment variables. For example, if you have diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 9f852dc7b..c08a1a943 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -693,25 +693,27 @@ public: This complements the [`system`](#conf-system) and [`extra-platforms`](#conf-extra-platforms) configuration options and the corresponding [`system`](@docroot@/language/derivations.md#attr-system) attribute on derivations. - Derivations can require system features in the derivation attribute [`requiredSystemFeatures`](@docroot@/language/advanced-attributes.md#adv-attr-requiredSystemFeatures). + A derivation can require system features in the [`requiredSystemFeatures` attribute](@docroot@/language/advanced-attributes.md#adv-attr-requiredSystemFeatures), and the machine to build the derivation must have them. - System features are generally user-defined, but the following have special treatment: + System features are user-defined, but Nix sets the following defaults: - `kvm` - Set by default if `/dev/kvm` is accessible. + Included by default if `/dev/kvm` is accessible. - `nixos-test`, `benchmark`, `big-parallel` - These historical pseudo-features are always enabled for backwards compatibility, used in Nixpkgs to route Hydra builds to specific machines. + These historical pseudo-features are always enabled for backwards compatibility, as they are used in Nixpkgs to route Hydra builds to specific machines. - `ca-derivations` - Set by default if the [`ca-derivations` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) is enabled. + Included by default if the [`ca-derivations` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) is enabled. + + This system feature is implicitly required by derivations with the [`__contentAddressed` attribute](@docroot@/language/advanced-attributes.md#adv-attr-__contentAddressed). - `recursive-nix` - Set by default if the [`recursive-nix` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-recursive-nix) is enabled. + Included by default if the [`recursive-nix` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-recursive-nix) is enabled. - `uid-range` @@ -720,7 +722,7 @@ public: [nspawn]: https://github.com/NixOS/nix/blob/67bcb99700a0da1395fa063d7c6586740b304598/tests/systemd-nspawn.nix. - Set by default on Linux if the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting is enabled. + Included by default on Linux if the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting is enabled. )", {}, false}; Setting substituters{ From c4dbb55ba93c9c81d798faa6a19285ebe2717a25 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 11 Aug 2023 17:22:55 +0200 Subject: [PATCH 011/139] initLibUtil: Add exception handling self-check --- src/libutil/error.cc | 5 +++++ src/libutil/error.hh | 4 ++++ src/libutil/util.cc | 17 +++++++++++++++++ 3 files changed, 26 insertions(+) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index c9d61942a..4a1b346ef 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -14,6 +14,11 @@ void BaseError::addTrace(std::shared_ptr && e, hintformat hint, boo err.traces.push_front(Trace { .pos = std::move(e), .hint = hint, .frame = frame }); } +void throwExceptionSelfCheck(){ + // This is meant to be caught in initLibUtil() + throw SysError("C++ exception handling is broken. This would appear to be a problem with the way Nix was compiled and/or linked and/or loaded."); +} + // c++ std::exception descendants must have a 'const char* what()' function. // This stringifies the error and caches it for use by what(), or similarly by msg(). const std::string & BaseError::calcWhat() const diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 6a0923081..be5c5e252 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -214,4 +214,8 @@ public: } }; +/** Throw an exception for the purpose of checking that exception handling works; see 'initLibUtil()'. + */ +void throwExceptionSelfCheck(); + } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 26f9dc8a8..f24a6e165 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -48,6 +48,23 @@ extern char * * environ __attribute__((weak)); namespace nix { void initLibUtil() { + // Check that exception handling works. Exception handling has been observed + // not to work on darwin when the linker flags aren't quite right. + // In this case we don't want to expose the user to some unrelated uncaught + // exception, but rather tell them exactly that exception handling is + // broken. + // When exception handling fails, the message tends to be printed by the + // C++ runtime, followed by an abort. + // For example on macOS we might see an error such as + // libc++abi: terminating with uncaught exception of type nix::SysError: error: C++ exception handling is broken. This would appear to be a problem with the way Nix was compiled and/or linked and/or loaded. + bool caught = false; + try { + throwExceptionSelfCheck(); + } catch (nix::Error _e) { + caught = true; + } + // This is not actually the main point of this check, but let's make sure anyway: + assert(caught); } std::optional getEnv(const std::string & key) From 1ef8008ca7a7b38570f06abdbb7960e93ac044ef Mon Sep 17 00:00:00 2001 From: Alex Zero Date: Sun, 13 Aug 2023 01:31:32 +0100 Subject: [PATCH 012/139] Fix follow path checking at depths greater than 2 We need to recurse into the input tree to handle follows paths that trarverse multiple inputs that may or may not be follow paths themselves. --- src/libexpr/flake/lockfile.cc | 2 +- tests/flakes/follow-paths.sh | 63 +++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index ba2fd46f0..3c202967a 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -345,7 +345,7 @@ void LockFile::check() for (auto & [inputPath, input] : inputs) { if (auto follows = std::get_if<1>(&input)) { - if (!follows->empty() && !get(inputs, *follows)) + if (!follows->empty() && !findInput(*follows)) throw Error("input '%s' follows a non-existent input '%s'", printInputPath(inputPath), printInputPath(*follows)); diff --git a/tests/flakes/follow-paths.sh b/tests/flakes/follow-paths.sh index fe9b51c65..a70d9acb2 100644 --- a/tests/flakes/follow-paths.sh +++ b/tests/flakes/follow-paths.sh @@ -148,3 +148,66 @@ git -C $flakeFollowsA add flake.nix nix flake lock $flakeFollowsA 2>&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid'" nix flake lock $flakeFollowsA 2>&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid2'" + +# Now test follow path overloading +flakeFollowsOverloadA=$TEST_ROOT/follows/overload/flakeA +flakeFollowsOverloadB=$TEST_ROOT/follows/overload/flakeA/flakeB +flakeFollowsOverloadC=$TEST_ROOT/follows/overload/flakeA/flakeB/flakeC +flakeFollowsOverloadD=$TEST_ROOT/follows/overload/flakeA/flakeB/flakeC/flakeD + +# Test following path flakerefs. +createGitRepo $flakeFollowsOverloadA +mkdir -p $flakeFollowsOverloadB +mkdir -p $flakeFollowsOverloadC +mkdir -p $flakeFollowsOverloadD + +cat > $flakeFollowsOverloadD/flake.nix < $flakeFollowsOverloadC/flake.nix < $flakeFollowsOverloadB/flake.nix < $flakeFollowsOverloadA/flake.nix < Date: Mon, 14 Aug 2023 18:52:21 +0100 Subject: [PATCH 013/139] Add release notes for the previous commit --- doc/manual/src/release-notes/rl-next.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 7ddb5ca00..5a870f1ab 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -22,3 +22,5 @@ - Introduce a new [`outputOf`](@docroot@/language/builtins.md#builtins-outputOf) builtin. It is part of the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. + +- Flake follow paths at depths greater than 2 are now handled correctly, preventing "follows a non-existent input" errors. From 20d9c672d14de3f05a791251a67618d679cf4173 Mon Sep 17 00:00:00 2001 From: Vertex <5567402+VertexA115@users.noreply.github.com> Date: Tue, 15 Aug 2023 10:10:27 +0100 Subject: [PATCH 014/139] Update tests/flakes/follow-paths.sh Co-authored-by: Robert Hensing --- tests/flakes/follow-paths.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/flakes/follow-paths.sh b/tests/flakes/follow-paths.sh index a70d9acb2..f9d5c8c80 100644 --- a/tests/flakes/follow-paths.sh +++ b/tests/flakes/follow-paths.sh @@ -150,6 +150,25 @@ nix flake lock $flakeFollowsA 2>&1 | grep "warning: input 'B' has an override fo nix flake lock $flakeFollowsA 2>&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid2'" # Now test follow path overloading +# This tests a lockfile checking regression https://github.com/NixOS/nix/pull/8819 +# +# We construct the following graph, where p->q means p has input q. +# A double edge means that the edge gets overridden using `follows`. +# +# A +# / \ +# / \ +# v v +# B ==> C --- follows declared in A +# \\ / +# \\/ --- follows declared in B +# v +# D +# +# The message was +# error: input 'B/D' follows a non-existent input 'B/C/D' +# +# Note that for `B` to resolve its follow for `D`, it needs `C/D`, for which it needs to resolve the follow on `C` first. flakeFollowsOverloadA=$TEST_ROOT/follows/overload/flakeA flakeFollowsOverloadB=$TEST_ROOT/follows/overload/flakeA/flakeB flakeFollowsOverloadC=$TEST_ROOT/follows/overload/flakeA/flakeB/flakeC From 75243c96937b472665b94729df81f8296b262085 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 18 Aug 2023 14:46:13 +0200 Subject: [PATCH 015/139] test/flakes/follow-paths.sh: Quote Co-authored-by: Alex Ameen --- tests/flakes/follow-paths.sh | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/flakes/follow-paths.sh b/tests/flakes/follow-paths.sh index f9d5c8c80..dc97027ac 100644 --- a/tests/flakes/follow-paths.sh +++ b/tests/flakes/follow-paths.sh @@ -146,8 +146,8 @@ EOF git -C $flakeFollowsA add flake.nix -nix flake lock $flakeFollowsA 2>&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid'" -nix flake lock $flakeFollowsA 2>&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid2'" +nix flake lock "$flakeFollowsA" 2>&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid'" +nix flake lock "$flakeFollowsA" 2>&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid2'" # Now test follow path overloading # This tests a lockfile checking regression https://github.com/NixOS/nix/pull/8819 @@ -169,18 +169,18 @@ nix flake lock $flakeFollowsA 2>&1 | grep "warning: input 'B' has an override fo # error: input 'B/D' follows a non-existent input 'B/C/D' # # Note that for `B` to resolve its follow for `D`, it needs `C/D`, for which it needs to resolve the follow on `C` first. -flakeFollowsOverloadA=$TEST_ROOT/follows/overload/flakeA -flakeFollowsOverloadB=$TEST_ROOT/follows/overload/flakeA/flakeB -flakeFollowsOverloadC=$TEST_ROOT/follows/overload/flakeA/flakeB/flakeC -flakeFollowsOverloadD=$TEST_ROOT/follows/overload/flakeA/flakeB/flakeC/flakeD +flakeFollowsOverloadA="$TEST_ROOT/follows/overload/flakeA" +flakeFollowsOverloadB="$TEST_ROOT/follows/overload/flakeA/flakeB" +flakeFollowsOverloadC="$TEST_ROOT/follows/overload/flakeA/flakeB/flakeC" +flakeFollowsOverloadD="$TEST_ROOT/follows/overload/flakeA/flakeB/flakeC/flakeD" # Test following path flakerefs. -createGitRepo $flakeFollowsOverloadA -mkdir -p $flakeFollowsOverloadB -mkdir -p $flakeFollowsOverloadC -mkdir -p $flakeFollowsOverloadD +createGitRepo "$flakeFollowsOverloadA" +mkdir -p "$flakeFollowsOverloadB" +mkdir -p "$flakeFollowsOverloadC" +mkdir -p "$flakeFollowsOverloadD" -cat > $flakeFollowsOverloadD/flake.nix < "$flakeFollowsOverloadD/flake.nix" < $flakeFollowsOverloadD/flake.nix < $flakeFollowsOverloadC/flake.nix < "$flakeFollowsOverloadC/flake.nix" < $flakeFollowsOverloadC/flake.nix < $flakeFollowsOverloadB/flake.nix < "$flakeFollowsOverloadB/flake.nix" < $flakeFollowsOverloadB/flake.nix < $flakeFollowsOverloadA/flake.nix < "$flakeFollowsOverloadA/flake.nix" < $flakeFollowsOverloadA/flake.nix < Date: Wed, 16 Aug 2023 12:04:15 -0400 Subject: [PATCH 016/139] Delete `EvalState::addToSearchPath` This function is now trivial enough that it doesn't need to exist. `EvalState` can still be initialized with a custom search path, but we don't have a need to mutate the search path after it has been constructed, and I don't see why we would need to in the future. Fixes #8229 --- src/libexpr/eval.cc | 4 ++-- src/libexpr/eval.hh | 2 -- src/libexpr/parser.y | 6 ------ 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7e839dbe7..6d445fd96 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -527,9 +527,9 @@ EvalState::EvalState( /* Initialise the Nix expression search path. */ if (!evalSettings.pureEval) { for (auto & i : _searchPath.elements) - addToSearchPath(SearchPath::Elem {i}); + searchPath.elements.emplace_back(SearchPath::Elem {i}); for (auto & i : evalSettings.nixPath.get()) - addToSearchPath(SearchPath::Elem::parse(i)); + searchPath.elements.emplace_back(SearchPath::Elem::parse(i)); } if (evalSettings.restrictEval || evalSettings.pureEval) { diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 0c3eb6505..fa8fa462b 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -341,8 +341,6 @@ public: std::shared_ptr buildStore = nullptr); ~EvalState(); - void addToSearchPath(SearchPath::Elem && elem); - SearchPath getSearchPath() { return searchPath; } /** diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 201370b90..792f51fde 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -736,12 +736,6 @@ Expr * EvalState::parseStdin() } -void EvalState::addToSearchPath(SearchPath::Elem && elem) -{ - searchPath.elements.emplace_back(std::move(elem)); -} - - SourcePath EvalState::findFile(const std::string_view path) { return findFile(searchPath, path); From 8130373be9a10681a94b0546242843db158f30d3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Aug 2023 22:22:21 +0000 Subject: [PATCH 017/139] Bump zeebe-io/backport-action from 1.3.1 to 1.4.0 Bumps [zeebe-io/backport-action](https://github.com/zeebe-io/backport-action) from 1.3.1 to 1.4.0. - [Release notes](https://github.com/zeebe-io/backport-action/releases) - [Commits](https://github.com/zeebe-io/backport-action/compare/v1.3.1...v1.4.0) --- updated-dependencies: - dependency-name: zeebe-io/backport-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/backport.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 816474ed5..62bef6322 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -21,7 +21,7 @@ jobs: fetch-depth: 0 - name: Create backport PRs # should be kept in sync with `version` - uses: zeebe-io/backport-action@v1.3.1 + uses: zeebe-io/backport-action@v1.4.0 with: # Config README: https://github.com/zeebe-io/backport-action#backport-action github_token: ${{ secrets.GITHUB_TOKEN }} From 81045f243f91adcd44da0080c9ac83d2c4176125 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 22 Aug 2023 19:29:25 +0200 Subject: [PATCH 018/139] Tarball trees: Propagate lastModified This makes them behave consistently with GitHub/GitLab flakes. --- src/libfetchers/tarball.cc | 5 ++++- tests/tarball.sh | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 107d38e92..e3a41e75e 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -232,7 +232,7 @@ struct CurlInputScheme : InputScheme if (type != inputType()) return {}; // FIXME: some of these only apply to TarballInputScheme. - std::set allowedNames = {"type", "url", "narHash", "name", "unpack", "rev", "revCount"}; + std::set allowedNames = {"type", "url", "narHash", "name", "unpack", "rev", "revCount", "lastModified"}; for (auto & [name, value] : attrs) if (!allowedNames.count(name)) throw Error("unsupported %s input attribute '%s'", *type, name); @@ -310,6 +310,9 @@ struct TarballInputScheme : CurlInputScheme input = immutableInput; } + if (result.lastModified && !input.attrs.contains("lastModified")) + input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified)); + return {result.tree.storePath, std::move(input)}; } }; diff --git a/tests/tarball.sh b/tests/tarball.sh index 5f39658c9..6e621a28c 100644 --- a/tests/tarball.sh +++ b/tests/tarball.sh @@ -9,6 +9,7 @@ rm -rf $tarroot mkdir -p $tarroot cp dependencies.nix $tarroot/default.nix cp config.nix dependencies.builder*.sh $tarroot/ +touch -d '@1000000000' $tarroot $tarroot/* hash=$(nix hash path $tarroot) @@ -36,6 +37,8 @@ test_tarball() { nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file:///does-not-exist/must-remain-unused/$tarball; narHash = \"$hash\"; })" expectStderr 102 nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" | grep 'NAR hash mismatch in input' + [[ $(nix eval --impure --expr "(fetchTree file://$tarball).lastModified") = 1000000000 ]] + nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" >&2 nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" 2>&1 | grep 'true' From 4a435ad2288bffd5c28d90758e4e26160b53fb8c Mon Sep 17 00:00:00 2001 From: Uri Zafrir Date: Wed, 23 Aug 2023 18:18:25 +0300 Subject: [PATCH 019/139] Add introductory sentence to advanced topics (#8861) --- doc/manual/src/advanced-topics/advanced-topics.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/advanced-topics/advanced-topics.md b/doc/manual/src/advanced-topics/advanced-topics.md index 8b1378917..9a4d12a33 100644 --- a/doc/manual/src/advanced-topics/advanced-topics.md +++ b/doc/manual/src/advanced-topics/advanced-topics.md @@ -1 +1 @@ - +This section lists advanced topics related to builds and builds performance From 7d82341633d81f50f3e5e0b0896cfac5ea805d57 Mon Sep 17 00:00:00 2001 From: p01arst0rm Date: Wed, 23 Aug 2023 19:28:24 +0100 Subject: [PATCH 020/139] update system definitions --- flake.nix | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index 6e1f48cd7..cf7c1fa12 100644 --- a/flake.nix +++ b/flake.nix @@ -19,10 +19,12 @@ then "" else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}"; + linux32BitSystems = [ "i686-linux" ]; linux64BitSystems = [ "x86_64-linux" "aarch64-linux" ]; - linuxSystems = linux64BitSystems ++ [ "i686-linux" ]; - systems = linuxSystems ++ [ "x86_64-darwin" "aarch64-darwin" ]; - + linuxSystems = linux32BitSystems ++ linux64BitSystems; + darwinSystems = [ "x86_64-darwin" "aarch64-darwin" ]; + systems = linuxSystems ++ darwinSystems; + crossSystems = [ "armv6l-linux" "armv7l-linux" ]; stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" "libcxxStdenv" "ccacheStdenv" ]; From d5b130ef13bd9a77d803950ca814ac9a612f77b8 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 24 Aug 2023 10:00:17 +0200 Subject: [PATCH 021/139] glossary: dedent list and do not use forced line breaks this makes it slightly easier to work with and consistent with all the other markdown lists in use --- doc/manual/src/glossary.md | 367 ++++++++++++++++++++----------------- 1 file changed, 200 insertions(+), 167 deletions(-) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index ac0bb3c2f..d950a4ca8 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -1,236 +1,269 @@ # Glossary - - [derivation]{#gloss-derivation}\ - A description of a build task. The result of a derivation is a - store object. Derivations are typically specified in Nix expressions - using the [`derivation` primitive](./language/derivations.md). These are - translated into low-level *store derivations* (implicitly by - `nix-env` and `nix-build`, or explicitly by `nix-instantiate`). +- [derivation]{#gloss-derivation} - [derivation]: #gloss-derivation + A description of a build task. The result of a derivation is a + store object. Derivations are typically specified in Nix expressions + using the [`derivation` primitive](./language/derivations.md). These are + translated into low-level *store derivations* (implicitly by + `nix-env` and `nix-build`, or explicitly by `nix-instantiate`). - - [store derivation]{#gloss-store-derivation}\ - A [derivation] represented as a `.drv` file in the [store]. - It has a [store path], like any [store object]. + [derivation]: #gloss-derivation - Example: `/nix/store/g946hcz4c8mdvq2g8vxx42z51qb71rvp-git-2.38.1.drv` +- [store derivation]{#gloss-store-derivation} - See [`nix derivation show`](./command-ref/new-cli/nix3-derivation-show.md) (experimental) for displaying the contents of store derivations. + A [derivation] represented as a `.drv` file in the [store]. + It has a [store path], like any [store object]. - [store derivation]: #gloss-store-derivation + Example: `/nix/store/g946hcz4c8mdvq2g8vxx42z51qb71rvp-git-2.38.1.drv` - - [instantiate]{#gloss-instantiate}, instantiation\ - Translate a [derivation] into a [store derivation]. + See [`nix derivation show`](./command-ref/new-cli/nix3-derivation-show.md) (experimental) for displaying the contents of store derivations. - See [`nix-instantiate`](./command-ref/nix-instantiate.md). + [store derivation]: #gloss-store-derivation - [instantiate]: #gloss-instantiate +- [instantiate]{#gloss-instantiate}, instantiation - - [realise]{#gloss-realise}, realisation\ - Ensure a [store path] is [valid][validity]. + Translate a [derivation] into a [store derivation]. - This means either running the `builder` executable as specified in the corresponding [derivation] or fetching a pre-built [store object] from a [substituter]. + See [`nix-instantiate`](./command-ref/nix-instantiate.md). - See [`nix-build`](./command-ref/nix-build.md) and [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md). + [instantiate]: #gloss-instantiate - See [`nix build`](./command-ref/new-cli/nix3-build.md) (experimental). +- [realise]{#gloss-realise}, realisation - [realise]: #gloss-realise + Ensure a [store path] is [valid][validity]. - - [content-addressed derivation]{#gloss-content-addressed-derivation}\ - A derivation which has the - [`__contentAddressed`](./language/advanced-attributes.md#adv-attr-__contentAddressed) - attribute set to `true`. + This means either running the `builder` executable as specified in the corresponding [derivation] or fetching a pre-built [store object] from a [substituter]. - - [fixed-output derivation]{#gloss-fixed-output-derivation}\ - A derivation which includes the - [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute. + See [`nix-build`](./command-ref/nix-build.md) and [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md). - - [store]{#gloss-store}\ - The location in the file system where store objects live. Typically - `/nix/store`. + See [`nix build`](./command-ref/new-cli/nix3-build.md) (experimental). - From the perspective of the location where Nix is - invoked, the Nix store can be referred to - as a "_local_" or a "_remote_" one: + [realise]: #gloss-realise - + A [local store]{#gloss-local-store} exists on the filesystem of - the machine where Nix is invoked. You can use other - local stores by passing the `--store` flag to the - `nix` command. Local stores can be used for building derivations. +- [content-addressed derivation]{#gloss-content-addressed-derivation} - + A *remote store* exists anywhere other than the - local filesystem. One example is the `/nix/store` - directory on another machine, accessed via `ssh` or - served by the `nix-serve` Perl script. + A derivation which has the + [`__contentAddressed`](./language/advanced-attributes.md#adv-attr-__contentAddressed) + attribute set to `true`. - [store]: #gloss-store - [local store]: #gloss-local-store +- [fixed-output derivation]{#gloss-fixed-output-derivation} - - [chroot store]{#gloss-chroot-store}\ - A [local store] whose canonical path is anything other than `/nix/store`. + A derivation which includes the + [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute. - - [binary cache]{#gloss-binary-cache}\ - A *binary cache* is a Nix store which uses a different format: its - metadata and signatures are kept in `.narinfo` files rather than in a - [Nix database]. This different format simplifies serving store objects - over the network, but cannot host builds. Examples of binary caches - include S3 buckets and the [NixOS binary cache](https://cache.nixos.org). +- [store]{#gloss-store} - - [store path]{#gloss-store-path}\ - The location of a [store object] in the file system, i.e., an - immediate child of the Nix store directory. + The location in the file system where store objects live. Typically + `/nix/store`. - Example: `/nix/store/a040m110amc4h71lds2jmr8qrkj2jhxd-git-2.38.1` + From the perspective of the location where Nix is + invoked, the Nix store can be referred to + as a "_local_" or a "_remote_" one: - [store path]: #gloss-store-path + + A [local store]{#gloss-local-store} exists on the filesystem of + the machine where Nix is invoked. You can use other + local stores by passing the `--store` flag to the + `nix` command. Local stores can be used for building derivations. - - [file system object]{#gloss-store-object}\ - The Nix data model for representing simplified file system data. + + A *remote store* exists anywhere other than the + local filesystem. One example is the `/nix/store` + directory on another machine, accessed via `ssh` or + served by the `nix-serve` Perl script. - See [File System Object](@docroot@/architecture/file-system-object.md) for details. + [store]: #gloss-store + [local store]: #gloss-local-store - [file system object]: #gloss-file-system-object +- [chroot store]{#gloss-chroot-store} - - [store object]{#gloss-store-object}\ + A [local store] whose canonical path is anything other than `/nix/store`. - A store object consists of a [file system object], [reference]s to other store objects, and other metadata. - It can be referred to by a [store path]. +- [binary cache]{#gloss-binary-cache} - [store object]: #gloss-store-object + A *binary cache* is a Nix store which uses a different format: its + metadata and signatures are kept in `.narinfo` files rather than in a + [Nix database]. This different format simplifies serving store objects + over the network, but cannot host builds. Examples of binary caches + include S3 buckets and the [NixOS binary cache](https://cache.nixos.org). - - [input-addressed store object]{#gloss-input-addressed-store-object}\ - A store object produced by building a - non-[content-addressed](#gloss-content-addressed-derivation), - non-[fixed-output](#gloss-fixed-output-derivation) - derivation. +- [store path]{#gloss-store-path} - - [output-addressed store object]{#gloss-output-addressed-store-object}\ - A [store object] whose [store path] is determined by its contents. - This includes derivations, the outputs of [content-addressed derivations](#gloss-content-addressed-derivation), and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation). + The location of a [store object] in the file system, i.e., an + immediate child of the Nix store directory. - - [substitute]{#gloss-substitute}\ - A substitute is a command invocation stored in the [Nix database] that - describes how to build a store object, bypassing the normal build - mechanism (i.e., derivations). Typically, the substitute builds the - store object by downloading a pre-built version of the store object - from some server. + Example: `/nix/store/a040m110amc4h71lds2jmr8qrkj2jhxd-git-2.38.1` - - [substituter]{#gloss-substituter}\ - An additional [store]{#gloss-store} from which Nix can obtain store objects instead of building them. - Often the substituter is a [binary cache](#gloss-binary-cache), but any store can serve as substituter. + [store path]: #gloss-store-path - See the [`substituters` configuration option](./command-ref/conf-file.md#conf-substituters) for details. +- [file system object]{#gloss-store-object} - [substituter]: #gloss-substituter + The Nix data model for representing simplified file system data. - - [purity]{#gloss-purity}\ - The assumption that equal Nix derivations when run always produce - the same output. This cannot be guaranteed in general (e.g., a - builder can rely on external inputs such as the network or the - system time) but the Nix model assumes it. + See [File System Object](@docroot@/architecture/file-system-object.md) for details. - - [Nix database]{#gloss-nix-database}\ - An SQlite database to track [reference]s between [store object]s. - This is an implementation detail of the [local store]. + [file system object]: #gloss-file-system-object - Default location: `/nix/var/nix/db`. +- [store object]{#gloss-store-object} - [Nix database]: #gloss-nix-database - - [Nix expression]{#gloss-nix-expression}\ - A high-level description of software packages and compositions - thereof. Deploying software using Nix entails writing Nix - expressions for your packages. Nix expressions are translated to - derivations that are stored in the Nix store. These derivations can - then be built. + A store object consists of a [file system object], [reference]s to other store objects, and other metadata. + It can be referred to by a [store path]. - - [reference]{#gloss-reference}\ - A [store object] `O` is said to have a *reference* to a store object `P` if a [store path] to `P` appears in the contents of `O`. + [store object]: #gloss-store-object - Store objects can refer to both other store objects and themselves. - References from a store object to itself are called *self-references*. - References other than a self-reference must not form a cycle. +- [input-addressed store object]{#gloss-input-addressed-store-object} - [reference]: #gloss-reference + A store object produced by building a + non-[content-addressed](#gloss-content-addressed-derivation), + non-[fixed-output](#gloss-fixed-output-derivation) + derivation. - - [reachable]{#gloss-reachable}\ - A store path `Q` is reachable from another store path `P` if `Q` - is in the *closure* of the *references* relation. +- [output-addressed store object]{#gloss-output-addressed-store-object} - - [closure]{#gloss-closure}\ - The closure of a store path is the set of store paths that are - directly or indirectly “reachable” from that store path; that is, - it’s the closure of the path under the *references* relation. For - a package, the closure of its derivation is equivalent to the - build-time dependencies, while the closure of its output path is - equivalent to its runtime dependencies. For correct deployment it - is necessary to deploy whole closures, since otherwise at runtime - files could be missing. The command `nix-store --query --requisites ` prints out - closures of store paths. + A [store object] whose [store path] is determined by its contents. + This includes derivations, the outputs of [content-addressed derivations](#gloss-content-addressed-derivation), and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation). - As an example, if the [store object] at path `P` contains a [reference] - to a store object at path `Q`, then `Q` is in the closure of `P`. Further, if `Q` - references `R` then `R` is also in the closure of `P`. +- [substitute]{#gloss-substitute} - [closure]: #gloss-closure + A substitute is a command invocation stored in the [Nix database] that + describes how to build a store object, bypassing the normal build + mechanism (i.e., derivations). Typically, the substitute builds the + store object by downloading a pre-built version of the store object + from some server. - - [output path]{#gloss-output-path}\ - A [store path] produced by a [derivation]. +- [substituter]{#gloss-substituter} - [output path]: #gloss-output-path + An additional [store]{#gloss-store} from which Nix can obtain store objects instead of building them. + Often the substituter is a [binary cache](#gloss-binary-cache), but any store can serve as substituter. - - [deriver]{#gloss-deriver}\ - The [store derivation] that produced an [output path]. + See the [`substituters` configuration option](./command-ref/conf-file.md#conf-substituters) for details. - - [validity]{#gloss-validity}\ - A store path is valid if all [store object]s in its [closure] can be read from the [store]. + [substituter]: #gloss-substituter - For a [local store], this means: - - The store path leads to an existing [store object] in that [store]. - - The store path is listed in the [Nix database] as being valid. - - All paths in the store path's [closure] are valid. +- [purity]{#gloss-purity} - [validity]: #gloss-validity + The assumption that equal Nix derivations when run always produce + the same output. This cannot be guaranteed in general (e.g., a + builder can rely on external inputs such as the network or the + system time) but the Nix model assumes it. - - [user environment]{#gloss-user-env}\ - An automatically generated store object that consists of a set of - symlinks to “active” applications, i.e., other store paths. These - are generated automatically by - [`nix-env`](./command-ref/nix-env.md). See *profiles*. +- [Nix database]{#gloss-nix-database} - - [profile]{#gloss-profile}\ - A symlink to the current *user environment* of a user, e.g., - `/nix/var/nix/profiles/default`. + An SQlite database to track [reference]s between [store object]s. + This is an implementation detail of the [local store]. - - [installable]{#gloss-installable}\ - Something that can be realised in the Nix store. + Default location: `/nix/var/nix/db`. - See [installables](./command-ref/new-cli/nix.md#installables) for [`nix` commands](./command-ref/new-cli/nix.md) (experimental) for details. + [Nix database]: #gloss-nix-database - - [NAR]{#gloss-nar}\ - A *N*ix *AR*chive. This is a serialisation of a path in the Nix - store. It can contain regular files, directories and symbolic - links. NARs are generated and unpacked using `nix-store --dump` - and `nix-store --restore`. +- [Nix expression]{#gloss-nix-expression} - - [`∅`]{#gloss-emtpy-set}\ - The empty set symbol. In the context of profile history, this denotes a package is not present in a particular version of the profile. + A high-level description of software packages and compositions + thereof. Deploying software using Nix entails writing Nix + expressions for your packages. Nix expressions are translated to + derivations that are stored in the Nix store. These derivations can + then be built. - - [`ε`]{#gloss-epsilon}\ - The epsilon symbol. In the context of a package, this means the version is empty. More precisely, the derivation does not have a version attribute. +- [reference]{#gloss-reference} - - [string interpolation]{#gloss-string-interpolation}\ - Expanding expressions enclosed in `${ }` within a [string], [path], or [attribute name]. + A [store object] `O` is said to have a *reference* to a store object `P` if a [store path] to `P` appears in the contents of `O`. - See [String interpolation](./language/string-interpolation.md) for details. + Store objects can refer to both other store objects and themselves. + References from a store object to itself are called *self-references*. + References other than a self-reference must not form a cycle. - [string]: ./language/values.md#type-string - [path]: ./language/values.md#type-path - [attribute name]: ./language/values.md#attribute-set + [reference]: #gloss-reference - - [experimental feature]{#gloss-experimental-feature}\ - Not yet stabilized functionality guarded by named experimental feature flags. - These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting. +- [reachable]{#gloss-reachable} - See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/contributing/experimental-features.md). + A store path `Q` is reachable from another store path `P` if `Q` + is in the *closure* of the *references* relation. + +- [closure]{#gloss-closure} + + The closure of a store path is the set of store paths that are + directly or indirectly “reachable” from that store path; that is, + it’s the closure of the path under the *references* relation. For + a package, the closure of its derivation is equivalent to the + build-time dependencies, while the closure of its output path is + equivalent to its runtime dependencies. For correct deployment it + is necessary to deploy whole closures, since otherwise at runtime + files could be missing. The command `nix-store --query --requisites ` prints out + closures of store paths. + + As an example, if the [store object] at path `P` contains a [reference] + to a store object at path `Q`, then `Q` is in the closure of `P`. Further, if `Q` + references `R` then `R` is also in the closure of `P`. + + [closure]: #gloss-closure + +- [output path]{#gloss-output-path} + + A [store path] produced by a [derivation]. + + [output path]: #gloss-output-path + +- [deriver]{#gloss-deriver} + + The [store derivation] that produced an [output path]. + +- [validity]{#gloss-validity} + + A store path is valid if all [store object]s in its [closure] can be read from the [store]. + + For a [local store], this means: + - The store path leads to an existing [store object] in that [store]. + - The store path is listed in the [Nix database] as being valid. + - All paths in the store path's [closure] are valid. + + [validity]: #gloss-validity + +- [user environment]{#gloss-user-env} + + An automatically generated store object that consists of a set of + symlinks to “active” applications, i.e., other store paths. These + are generated automatically by + [`nix-env`](./command-ref/nix-env.md). See *profiles*. + +- [profile]{#gloss-profile} + + A symlink to the current *user environment* of a user, e.g., + `/nix/var/nix/profiles/default`. + +- [installable]{#gloss-installable} + + Something that can be realised in the Nix store. + + See [installables](./command-ref/new-cli/nix.md#installables) for [`nix` commands](./command-ref/new-cli/nix.md) (experimental) for details. + +- [NAR]{#gloss-nar} + + A *N*ix *AR*chive. This is a serialisation of a path in the Nix + store. It can contain regular files, directories and symbolic + links. NARs are generated and unpacked using `nix-store --dump` + and `nix-store --restore`. + +- [`∅`]{#gloss-emtpy-set} + + The empty set symbol. In the context of profile history, this denotes a package is not present in a particular version of the profile. + +- [`ε`]{#gloss-epsilon} + + The epsilon symbol. In the context of a package, this means the version is empty. More precisely, the derivation does not have a version attribute. + +- [string interpolation]{#gloss-string-interpolation} + + Expanding expressions enclosed in `${ }` within a [string], [path], or [attribute name]. + + See [String interpolation](./language/string-interpolation.md) for details. + + [string]: ./language/values.md#type-string + [path]: ./language/values.md#type-path + [attribute name]: ./language/values.md#attribute-set + +- [experimental feature]{#gloss-experimental-feature} + + Not yet stabilized functionality guarded by named experimental feature flags. + These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting. + + See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/contributing/experimental-features.md). From 925a444b925590df90e19d3c0071936f87d2b43d Mon Sep 17 00:00:00 2001 From: Guillaume Girol Date: Wed, 23 Aug 2023 12:00:00 +0000 Subject: [PATCH 022/139] add nix-store --query --valid-derivers command notably useful when nix-store --query --deriver returns a non-existing path. Co-authored-by: Felix Uhl --- doc/manual/src/command-ref/nix-store/query.md | 14 +++++++++++--- doc/manual/src/release-notes/rl-next.md | 4 ++++ src/nix-store/nix-store.cc | 18 +++++++++++++++++- tests/check-refs.nix | 2 +- tests/dependencies.nix | 13 +++++++++++++ tests/dependencies.sh | 17 +++++++++++++++++ tests/dyn-drv/eval-outputOf.sh | 4 ++-- tests/export-graph.nix | 4 ++-- tests/remote-store.sh | 2 +- 9 files changed, 68 insertions(+), 10 deletions(-) diff --git a/doc/manual/src/command-ref/nix-store/query.md b/doc/manual/src/command-ref/nix-store/query.md index cd45a4932..a158c76aa 100644 --- a/doc/manual/src/command-ref/nix-store/query.md +++ b/doc/manual/src/command-ref/nix-store/query.md @@ -5,8 +5,8 @@ # Synopsis `nix-store` {`--query` | `-q`} - {`--outputs` | `--requisites` | `-R` | `--references` | - `--referrers` | `--referrers-closure` | `--deriver` | `-d` | + {`--outputs` | `--requisites` | `-R` | `--references` | `--referrers` | + `--referrers-closure` | `--deriver` | `-d` | `--valid-derivers` | `--graph` | `--tree` | `--binding` *name* | `-b` *name* | `--hash` | `--size` | `--roots`} [`--use-output`] [`-u`] [`--force-realise`] [`-f`] @@ -82,13 +82,21 @@ symlink. in the Nix store that are dependent on *paths*. - `--deriver`; `-d`\ - Prints the [deriver] of the store paths *paths*. If + Prints the [deriver] that was used to build the store paths *paths*. If the path has no deriver (e.g., if it is a source file), or if the deriver is not known (e.g., in the case of a binary-only deployment), the string `unknown-deriver` is printed. + The returned deriver is not guaranteed to exist in the local store, for + example when *paths* were substituted from a binary cache. + Use `--valid-derivers` instead to obtain valid paths only. [deriver]: ../../glossary.md#gloss-deriver + - `--valid-derivers`\ + Prints a set of derivation files (`.drv`) which are supposed produce + said paths when realized. Might print nothing, for example for source paths + or paths subsituted from a binary cache. + - `--graph`\ Prints the references graph of the store paths *paths* in the format of the `dot` tool of AT\&T's [Graphviz diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 7ddb5ca00..66fe0c427 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -22,3 +22,7 @@ - Introduce a new [`outputOf`](@docroot@/language/builtins.md#builtins-outputOf) builtin. It is part of the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. + +- [`nix-store --query`](@docroot@/command-ref/nix-store/query.md) gained a new type of query: `--valid-derivers`. It returns all `.drv` files in the local store that *can be* used to build the output passed in argument. +This is in contrast to `--deriver`, which returns the single `.drv` file that *was actually* used to build the output passed in argument. In case the output was substituted from a binary cache, +this `.drv` file may only exist on said binary cache and not locally. diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 94956df66..96c3f7d7e 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -283,7 +283,7 @@ static void opQuery(Strings opFlags, Strings opArgs) { enum QueryType { qOutputs, qRequisites, qReferences, qReferrers - , qReferrersClosure, qDeriver, qBinding, qHash, qSize + , qReferrersClosure, qDeriver, qValidDerivers, qBinding, qHash, qSize , qTree, qGraph, qGraphML, qResolve, qRoots }; std::optional query; bool useOutput = false; @@ -299,6 +299,7 @@ static void opQuery(Strings opFlags, Strings opArgs) else if (i == "--referrers" || i == "--referers") query = qReferrers; else if (i == "--referrers-closure" || i == "--referers-closure") query = qReferrersClosure; else if (i == "--deriver" || i == "-d") query = qDeriver; + else if (i == "--valid-derivers") query = qValidDerivers; else if (i == "--binding" || i == "-b") { if (opArgs.size() == 0) throw UsageError("expected binding name"); @@ -372,6 +373,21 @@ static void opQuery(Strings opFlags, Strings opArgs) } break; + case qValidDerivers: { + StorePathSet result; + for (auto & i : opArgs) { + auto derivers = store->queryValidDerivers(store->followLinksToStorePath(i)); + for (const auto &i: derivers) { + result.insert(i); + } + } + auto sorted = store->topoSortPaths(result); + for (StorePaths::reverse_iterator i = sorted.rbegin(); + i != sorted.rend(); ++i) + cout << fmt("%s\n", store->printStorePath(*i)); + break; + } + case qBinding: for (auto & i : opArgs) { auto path = useDeriver(store->followLinksToStorePath(i)); diff --git a/tests/check-refs.nix b/tests/check-refs.nix index 99d69a226..89690e456 100644 --- a/tests/check-refs.nix +++ b/tests/check-refs.nix @@ -2,7 +2,7 @@ with import ./config.nix; rec { - dep = import ./dependencies.nix; + dep = import ./dependencies.nix {}; makeTest = nr: args: mkDerivation ({ name = "check-refs-" + toString nr; diff --git a/tests/dependencies.nix b/tests/dependencies.nix index 45aca1793..be1a7ae9a 100644 --- a/tests/dependencies.nix +++ b/tests/dependencies.nix @@ -1,3 +1,4 @@ +{ hashInvalidator ? "" }: with import ./config.nix; let { @@ -21,6 +22,17 @@ let { ''; }; + fod_input = mkDerivation { + name = "fod-input"; + buildCommand = '' + echo ${hashInvalidator} + echo FOD > $out + ''; + outputHashMode = "flat"; + outputHashAlgo = "sha256"; + outputHash = "1dq9p0hnm1y75q2x40fws5887bq1r840hzdxak0a9djbwvx0b16d"; + }; + body = mkDerivation { name = "dependencies-top"; builder = ./dependencies.builder0.sh + "/FOOBAR/../."; @@ -29,6 +41,7 @@ let { input1_drv = input1; input2_drv = input2; input0_drv = input0; + fod_input_drv = fod_input; meta.description = "Random test package"; }; diff --git a/tests/dependencies.sh b/tests/dependencies.sh index d5cd30396..b93dacac0 100644 --- a/tests/dependencies.sh +++ b/tests/dependencies.sh @@ -53,3 +53,20 @@ nix-store -q --referrers-closure "$input2OutPath" | grep "$outPath" # Check that the derivers are set properly. test $(nix-store -q --deriver "$outPath") = "$drvPath" nix-store -q --deriver "$input2OutPath" | grepQuiet -- "-input-2.drv" + +# --valid-derivers returns the currently single valid .drv file +test "$(nix-store -q --valid-derivers "$outPath")" = "$drvPath" + +# instantiate a different drv with the same output +drvPath2=$(nix-instantiate dependencies.nix --argstr hashInvalidator yay) + +# now --valid-derivers returns both +test "$(nix-store -q --valid-derivers "$outPath" | sort)" = "$(sort <<< "$drvPath"$'\n'"$drvPath2")" + +# check that nix-store --valid-derivers only returns existing drv +nix-store --delete "$drvPath" +test "$(nix-store -q --valid-derivers "$outPath")" = "$drvPath2" + +# check that --valid-derivers returns nothing when there are no valid derivers +nix-store --delete "$drvPath2" +test -z "$(nix-store -q --valid-derivers "$outPath")" diff --git a/tests/dyn-drv/eval-outputOf.sh b/tests/dyn-drv/eval-outputOf.sh index 99d917c06..9467feb8d 100644 --- a/tests/dyn-drv/eval-outputOf.sh +++ b/tests/dyn-drv/eval-outputOf.sh @@ -13,14 +13,14 @@ nix --experimental-features 'nix-command' eval --impure --expr \ # the future so it does work, but there are some design questions to # resolve first. Adding a test so we don't liberalise it by accident. expectStderr 1 nix --experimental-features 'nix-command dynamic-derivations' eval --impure --expr \ - 'builtins.outputOf (import ../dependencies.nix) "out"' \ + 'builtins.outputOf (import ../dependencies.nix {}) "out"' \ | grepQuiet "value is a set while a string was expected" # Test that "DrvDeep" string contexts are not supported at this time # # Like the above, this is a restriction we could relax later. expectStderr 1 nix --experimental-features 'nix-command dynamic-derivations' eval --impure --expr \ - 'builtins.outputOf (import ../dependencies.nix).drvPath "out"' \ + 'builtins.outputOf (import ../dependencies.nix {}).drvPath "out"' \ | grepQuiet "has a context which refers to a complete source and binary closure. This is not supported at this time" # Test using `builtins.outputOf` with static derivations diff --git a/tests/export-graph.nix b/tests/export-graph.nix index fdac9583d..64fe36bd1 100644 --- a/tests/export-graph.nix +++ b/tests/export-graph.nix @@ -17,13 +17,13 @@ rec { foo."bar.runtimeGraph" = mkDerivation { name = "dependencies"; builder = builtins.toFile "build-graph-builder" "${printRefs}"; - exportReferencesGraph = ["refs" (import ./dependencies.nix)]; + exportReferencesGraph = ["refs" (import ./dependencies.nix {})]; }; foo."bar.buildGraph" = mkDerivation { name = "dependencies"; builder = builtins.toFile "build-graph-builder" "${printRefs}"; - exportReferencesGraph = ["refs" (import ./dependencies.nix).drvPath]; + exportReferencesGraph = ["refs" (import ./dependencies.nix {}).drvPath]; }; } diff --git a/tests/remote-store.sh b/tests/remote-store.sh index ea32a20d3..50e6f24b9 100644 --- a/tests/remote-store.sh +++ b/tests/remote-store.sh @@ -24,7 +24,7 @@ fi import ( mkDerivation { name = "foo"; - bla = import ./dependencies.nix; + bla = import ./dependencies.nix {}; buildCommand = " echo \\\"hi\\\" > $out "; From 2f5d3da8062ae58242a8de2bad470a66478edea4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 25 Aug 2023 09:53:12 -0400 Subject: [PATCH 023/139] Introduce `OutputName` and `OutputNameView` type aliases Hopefully they make the code easier to understand! --- src/libexpr/primops.cc | 2 +- src/libstore/build/local-derivation-goal.cc | 2 +- src/libstore/build/local-derivation-goal.hh | 2 +- src/libstore/derivations.cc | 12 +++++----- src/libstore/derivations.hh | 12 +++++----- src/libstore/derived-path.cc | 4 ++-- src/libstore/derived-path.hh | 4 ++-- src/libstore/downstream-placeholder.cc | 4 ++-- src/libstore/downstream-placeholder.hh | 4 ++-- src/libstore/outputs-spec.hh | 26 +++++++++++++++------ src/libstore/realisation.hh | 6 ++--- 11 files changed, 45 insertions(+), 33 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5067da449..6b99b91e4 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1843,7 +1843,7 @@ static void prim_outputOf(EvalState & state, const PosIdx pos, Value * * args, V { SingleDerivedPath drvPath = state.coerceToSingleDerivedPath(pos, *args[0], "while evaluating the first argument to builtins.outputOf"); - std::string_view outputName = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument to builtins.outputOf"); + OutputNameView outputName = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument to builtins.outputOf"); state.mkSingleDerivedPathString( SingleDerivedPath::Built { diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 78f943d1f..64b55ca6a 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -2955,7 +2955,7 @@ bool LocalDerivationGoal::isReadDesc(int fd) } -StorePath LocalDerivationGoal::makeFallbackPath(std::string_view outputName) +StorePath LocalDerivationGoal::makeFallbackPath(OutputNameView outputName) { return worker.store.makeStorePath( "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName), diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 8827bfca3..0a05081c7 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -297,7 +297,7 @@ struct LocalDerivationGoal : public DerivationGoal * @todo Add option to randomize, so we can audit whether our * rewrites caught everything */ - StorePath makeFallbackPath(std::string_view outputName); + StorePath makeFallbackPath(OutputNameView outputName); }; } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 0b8bdaf1c..dc32c3847 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -12,7 +12,7 @@ namespace nix { -std::optional DerivationOutput::path(const Store & store, std::string_view drvName, std::string_view outputName) const +std::optional DerivationOutput::path(const Store & store, std::string_view drvName, OutputNameView outputName) const { return std::visit(overloaded { [](const DerivationOutput::InputAddressed & doi) -> std::optional { @@ -36,7 +36,7 @@ std::optional DerivationOutput::path(const Store & store, std::string } -StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const +StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, OutputNameView outputName) const { return store.makeFixedOutputPathFromCA( outputPathName(drvName, outputName), @@ -466,7 +466,7 @@ bool isDerivation(std::string_view fileName) } -std::string outputPathName(std::string_view drvName, std::string_view outputName) { +std::string outputPathName(std::string_view drvName, OutputNameView outputName) { std::string res { drvName }; if (outputName != "out") { res += "-"; @@ -810,7 +810,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr } -std::string hashPlaceholder(const std::string_view outputName) +std::string hashPlaceholder(const OutputNameView outputName) { // FIXME: memoize? return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(Base32, false); @@ -963,7 +963,7 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const const Hash impureOutputHash = hashString(htSHA256, "impure"); nlohmann::json DerivationOutput::toJSON( - const Store & store, std::string_view drvName, std::string_view outputName) const + const Store & store, std::string_view drvName, OutputNameView outputName) const { nlohmann::json res = nlohmann::json::object(); std::visit(overloaded { @@ -990,7 +990,7 @@ nlohmann::json DerivationOutput::toJSON( DerivationOutput DerivationOutput::fromJSON( - const Store & store, std::string_view drvName, std::string_view outputName, + const Store & store, std::string_view drvName, OutputNameView outputName, const nlohmann::json & _json, const ExperimentalFeatureSettings & xpSettings) { diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index a92082089..106056f2d 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -55,7 +55,7 @@ struct DerivationOutput * @param drvName The name of the derivation this is an output of, without the `.drv`. * @param outputName The name of this output. */ - StorePath path(const Store & store, std::string_view drvName, std::string_view outputName) const; + StorePath path(const Store & store, std::string_view drvName, OutputNameView outputName) const; GENERATE_CMP(CAFixed, me->ca); }; @@ -132,19 +132,19 @@ struct DerivationOutput * the safer interface provided by * BasicDerivation::outputsAndOptPaths */ - std::optional path(const Store & store, std::string_view drvName, std::string_view outputName) const; + std::optional path(const Store & store, std::string_view drvName, OutputNameView outputName) const; nlohmann::json toJSON( const Store & store, std::string_view drvName, - std::string_view outputName) const; + OutputNameView outputName) const; /** * @param xpSettings Stop-gap to avoid globals during unit tests. */ static DerivationOutput fromJSON( const Store & store, std::string_view drvName, - std::string_view outputName, + OutputNameView outputName, const nlohmann::json & json, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); }; @@ -405,7 +405,7 @@ bool isDerivation(std::string_view fileName); * This is usually -, but is just when * the output name is "out". */ -std::string outputPathName(std::string_view drvName, std::string_view outputName); +std::string outputPathName(std::string_view drvName, OutputNameView outputName); /** @@ -499,7 +499,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr * own outputs without needing to use the hash of a derivation in * itself, making the hash near-impossible to calculate. */ -std::string hashPlaceholder(const std::string_view outputName); +std::string hashPlaceholder(const OutputNameView outputName); extern const Hash impureOutputHash; diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 3594b7570..47d784deb 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -167,7 +167,7 @@ void drvRequireExperiment( SingleDerivedPath::Built SingleDerivedPath::Built::parse( const Store & store, ref drv, - std::string_view output, + OutputNameView output, const ExperimentalFeatureSettings & xpSettings) { drvRequireExperiment(*drv, xpSettings); @@ -179,7 +179,7 @@ SingleDerivedPath::Built SingleDerivedPath::Built::parse( DerivedPath::Built DerivedPath::Built::parse( const Store & store, ref drv, - std::string_view outputsS, + OutputNameView outputsS, const ExperimentalFeatureSettings & xpSettings) { drvRequireExperiment(*drv, xpSettings); diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index ec30dac61..4d7033df2 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -42,7 +42,7 @@ struct SingleDerivedPath; */ struct SingleDerivedPathBuilt { ref drvPath; - std::string output; + OutputName output; /** * Get the store path this is ultimately derived from (by realising @@ -71,7 +71,7 @@ struct SingleDerivedPathBuilt { */ static SingleDerivedPathBuilt parse( const Store & store, ref drvPath, - std::string_view outputs, + OutputNameView outputs, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); nlohmann::json toJSON(Store & store) const; diff --git a/src/libstore/downstream-placeholder.cc b/src/libstore/downstream-placeholder.cc index d951b7b7d..7e3f7548d 100644 --- a/src/libstore/downstream-placeholder.cc +++ b/src/libstore/downstream-placeholder.cc @@ -11,7 +11,7 @@ std::string DownstreamPlaceholder::render() const DownstreamPlaceholder DownstreamPlaceholder::unknownCaOutput( const StorePath & drvPath, - std::string_view outputName, + OutputNameView outputName, const ExperimentalFeatureSettings & xpSettings) { xpSettings.require(Xp::CaDerivations); @@ -25,7 +25,7 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownCaOutput( DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation( const DownstreamPlaceholder & placeholder, - std::string_view outputName, + OutputNameView outputName, const ExperimentalFeatureSettings & xpSettings) { xpSettings.require(Xp::DynamicDerivations); diff --git a/src/libstore/downstream-placeholder.hh b/src/libstore/downstream-placeholder.hh index d58a2ac14..c911ecea2 100644 --- a/src/libstore/downstream-placeholder.hh +++ b/src/libstore/downstream-placeholder.hh @@ -58,7 +58,7 @@ public: */ static DownstreamPlaceholder unknownCaOutput( const StorePath & drvPath, - std::string_view outputName, + OutputNameView outputName, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); /** @@ -72,7 +72,7 @@ public: */ static DownstreamPlaceholder unknownDerivation( const DownstreamPlaceholder & drvPlaceholder, - std::string_view outputName, + OutputNameView outputName, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); /** diff --git a/src/libstore/outputs-spec.hh b/src/libstore/outputs-spec.hh index ae19f1040..1ef99a5fc 100644 --- a/src/libstore/outputs-spec.hh +++ b/src/libstore/outputs-spec.hh @@ -13,24 +13,36 @@ namespace nix { +/** + * An (owned) output name. Just a type alias used to make code more + * readible. + */ +typedef std::string OutputName; + +/** + * A borrowed output name. Just a type alias used to make code more + * readible. + */ +typedef std::string_view OutputNameView; + struct OutputsSpec { /** * A non-empty set of outputs, specified by name */ - struct Names : std::set { - using std::set::set; + struct Names : std::set { + using std::set::set; /* These need to be "inherited manually" */ - Names(const std::set & s) - : std::set(s) + Names(const std::set & s) + : std::set(s) { assert(!empty()); } /** * Needs to be "inherited manually" */ - Names(std::set && s) - : std::set(s) + Names(std::set && s) + : std::set(s) { assert(!empty()); } /* This set should always be non-empty, so we delete this @@ -57,7 +69,7 @@ struct OutputsSpec { */ OutputsSpec() = delete; - bool contains(const std::string & output) const; + bool contains(const OutputName & output) const; /** * Create a new OutputsSpec which is the union of this and that. diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 0548b30c1..559483ce3 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -34,7 +34,7 @@ struct DrvOutput { /** * The name of the output. */ - std::string outputName; + OutputName outputName; std::string to_string() const; @@ -84,7 +84,7 @@ struct Realisation { * Since these are the outputs of a single derivation, we know the * output names are unique so we can use them as the map key. */ -typedef std::map SingleDrvOutputs; +typedef std::map SingleDrvOutputs; /** * Collection type for multiple derivations' outputs' `Realisation`s. @@ -146,7 +146,7 @@ public: MissingRealisation(DrvOutput & outputId) : MissingRealisation(outputId.outputName, outputId.strHash()) {} - MissingRealisation(std::string_view drv, std::string outputName) + MissingRealisation(std::string_view drv, OutputName outputName) : Error( "cannot operate on output '%s' of the " "unbuilt derivation '%s'", outputName, From 1c4caef14b51dbb4f749c45311c8e2c9acb75a60 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 16 Aug 2023 00:05:35 -0400 Subject: [PATCH 024/139] Throw `MissingRealisation` not plain `Error` in both `resolveDerivedPath` Now we are consistent with the other `resolveDerivedPath`, and other such functions. --- src/libstore/misc.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 631213306..c043b9b93 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -399,8 +399,7 @@ StorePath resolveDerivedPath(Store & store, const SingleDerivedPath & req, Store store.printStorePath(drvPath), bfd.output); auto & optPath = outputPaths.at(bfd.output); if (!optPath) - throw Error("'%s' does not yet map to a known concrete store path", - bfd.to_string(store)); + throw MissingRealisation(bfd.drvPath->to_string(store), bfd.output); return *optPath; }, }, req.raw()); From 692074f7142fcf8ede1266b6d8cbbd5feaf3221f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 15 Jan 2023 17:47:24 -0500 Subject: [PATCH 025/139] Use `Worker::makeDerivationGoal` less We're about to split up `DerivationGoal` a bit. At that point `makeDerivationGoal` will mean something more specific than it does today. (Perhaps a future rename will make this clearer.) On the other hand, the more public `Worker::makeGoal` function will continue to work exactly as before. So by moving some call sites to use that instead, we preemptively avoid issues in the next step. --- src/libstore/build/derivation-goal.cc | 14 ++++++++++++-- src/libstore/build/entry-points.cc | 7 +++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 84da7f2e1..ea56c0288 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -380,7 +380,12 @@ void DerivationGoal::gaveUpOnSubstitution() worker.store.printStorePath(i.first)); } - addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal)); + addWaitee(worker.makeGoal( + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(i.first), + .outputs = i.second, + }, + buildMode == bmRepair ? bmRepair : bmNormal)); } /* Copy the input sources from the eval store to the build @@ -452,7 +457,12 @@ void DerivationGoal::repairClosure() if (drvPath2 == outputsToDrv.end()) addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i, Repair))); else - addWaitee(worker.makeDerivationGoal(drvPath2->second, OutputsSpec::All(), bmRepair)); + addWaitee(worker.makeGoal( + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(drvPath2->second), + .outputs = OutputsSpec::All { }, + }, + bmRepair)); } if (waitees.empty()) { diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index e941b4e65..f71fb35a6 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -124,8 +124,11 @@ void Store::repairPath(const StorePath & path) auto info = queryPathInfo(path); if (info->deriver && isValidPath(*info->deriver)) { goals.clear(); - // FIXME: Should just build the specific output we need. - goals.insert(worker.makeDerivationGoal(*info->deriver, OutputsSpec::All { }, bmRepair)); + goals.insert(worker.makeGoal(DerivedPath::Built { + .drvPath = makeConstantStorePathRef(*info->deriver), + // FIXME: Should just build the specific output we need. + .outputs = OutputsSpec::All { }, + }, bmRepair)); worker.run(goals); } else throw Error(worker.failingExitStatus(), "cannot repair path '%s'", printStorePath(path)); From 5e3986f59cb58f48186a49dcec7aa317b4787522 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 8 Mar 2021 16:24:49 -0500 Subject: [PATCH 026/139] Adapt scheduler to work with dynamic derivations To avoid dealing with an optional `drvPath` (because we might not know it yet) everywhere, make an `CreateDerivationAndRealiseGoal`. This goal just builds/substitutes the derivation file, and then kicks of a build for that obtained derivation; in other words it does the chaining of goals when the drv file is missing (as can already be the case) or computed (new case). This also means the `getDerivation` state can be removed from `DerivationGoal`, which makes the `BasicDerivation` / in memory case and `Derivation` / drv file file case closer together. The map type is factored out for clarity, and because we will soon hvae a second use for it (`Derivation` itself). Co-authored-by: Robert Hensing --- .../create-derivation-and-realise-goal.cc | 157 ++++++++++++++++++ .../create-derivation-and-realise-goal.hh | 96 +++++++++++ src/libstore/build/derivation-goal.cc | 22 +-- src/libstore/build/derivation-goal.hh | 15 +- .../build/drv-output-substitution-goal.hh | 4 +- src/libstore/build/entry-points.cc | 11 +- src/libstore/build/goal.cc | 2 +- src/libstore/build/goal.hh | 22 ++- src/libstore/build/substitution-goal.hh | 4 +- src/libstore/build/worker.cc | 118 ++++++++++--- src/libstore/build/worker.hh | 22 +++ src/libstore/derived-path-map.cc | 33 ++++ src/libstore/derived-path-map.hh | 73 ++++++++ tests/dyn-drv/build-built-drv.sh | 4 +- 14 files changed, 525 insertions(+), 58 deletions(-) create mode 100644 src/libstore/build/create-derivation-and-realise-goal.cc create mode 100644 src/libstore/build/create-derivation-and-realise-goal.hh create mode 100644 src/libstore/derived-path-map.cc create mode 100644 src/libstore/derived-path-map.hh diff --git a/src/libstore/build/create-derivation-and-realise-goal.cc b/src/libstore/build/create-derivation-and-realise-goal.cc new file mode 100644 index 000000000..b01042f00 --- /dev/null +++ b/src/libstore/build/create-derivation-and-realise-goal.cc @@ -0,0 +1,157 @@ +#include "create-derivation-and-realise-goal.hh" +#include "worker.hh" + +namespace nix { + +CreateDerivationAndRealiseGoal::CreateDerivationAndRealiseGoal(ref drvReq, + const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) + : Goal(worker, DerivedPath::Built { .drvPath = drvReq, .outputs = wantedOutputs }) + , drvReq(drvReq) + , wantedOutputs(wantedOutputs) + , buildMode(buildMode) +{ + state = &CreateDerivationAndRealiseGoal::getDerivation; + name = fmt( + "outer obtaining drv from '%s' and then building outputs %s", + drvReq->to_string(worker.store), + std::visit(overloaded { + [&](const OutputsSpec::All) -> std::string { + return "* (all of them)"; + }, + [&](const OutputsSpec::Names os) { + return concatStringsSep(", ", quoteStrings(os)); + }, + }, wantedOutputs.raw)); + trace("created outer"); + + worker.updateProgress(); +} + + +CreateDerivationAndRealiseGoal::~CreateDerivationAndRealiseGoal() +{ +} + + +static StorePath pathPartOfReq(const SingleDerivedPath & req) +{ + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) { + return bo.path; + }, + [&](const SingleDerivedPath::Built & bfd) { + return pathPartOfReq(*bfd.drvPath); + }, + }, req.raw()); +} + + +std::string CreateDerivationAndRealiseGoal::key() +{ + /* Ensure that derivations get built in order of their name, + i.e. a derivation named "aardvark" always comes before "baboon". And + substitution goals and inner derivation goals always happen before + derivation goals (due to "b$"). */ + return "c$" + std::string(pathPartOfReq(*drvReq).name()) + "$" + drvReq->to_string(worker.store); +} + + +void CreateDerivationAndRealiseGoal::timedOut(Error && ex) +{ +} + + +void CreateDerivationAndRealiseGoal::work() +{ + (this->*state)(); +} + + +void CreateDerivationAndRealiseGoal::addWantedOutputs(const OutputsSpec & outputs) +{ + /* If we already want all outputs, there is nothing to do. */ + auto newWanted = wantedOutputs.union_(outputs); + bool needRestart = !newWanted.isSubsetOf(wantedOutputs); + wantedOutputs = newWanted; + + if (!needRestart) return; + + if (!optDrvPath) + // haven't started steps where the outputs matter yet + return; + worker.makeDerivationGoal(*optDrvPath, outputs, buildMode); +} + + +void CreateDerivationAndRealiseGoal::getDerivation() +{ + trace("outer init"); + + /* The first thing to do is to make sure that the derivation + exists. If it doesn't, it may be created through a + substitute. */ + if (auto optDrvPath = [this]() -> std::optional { + if (buildMode != bmNormal) return std::nullopt; + + auto drvPath = StorePath::dummy; + try { + drvPath = resolveDerivedPath(worker.store, *drvReq); + } catch (MissingRealisation) { + return std::nullopt; + } + return worker.evalStore.isValidPath(drvPath) || worker.store.isValidPath(drvPath) + ? std::optional { drvPath } + : std::nullopt; + }()) { + trace(fmt("already have drv '%s' for '%s', can go straight to building", + worker.store.printStorePath(*optDrvPath), + drvReq->to_string(worker.store))); + + loadAndBuildDerivation(); + } else { + trace("need to obtain drv we want to build"); + + addWaitee(worker.makeGoal(DerivedPath::fromSingle(*drvReq))); + + state = &CreateDerivationAndRealiseGoal::loadAndBuildDerivation; + if (waitees.empty()) work(); + } +} + + +void CreateDerivationAndRealiseGoal::loadAndBuildDerivation() +{ + trace("outer load and build derivation"); + + if (nrFailed != 0) { + amDone(ecFailed, Error("cannot build missing derivation '%s'", drvReq->to_string(worker.store))); + return; + } + + StorePath drvPath = resolveDerivedPath(worker.store, *drvReq); + /* Build this step! */ + concreteDrvGoal = worker.makeDerivationGoal(drvPath, wantedOutputs, buildMode); + addWaitee(upcast_goal(concreteDrvGoal)); + state = &CreateDerivationAndRealiseGoal::buildDone; + optDrvPath = std::move(drvPath); + if (waitees.empty()) work(); +} + + +void CreateDerivationAndRealiseGoal::buildDone() +{ + trace("outer build done"); + + buildResult = upcast_goal(concreteDrvGoal)->getBuildResult(DerivedPath::Built { + .drvPath = drvReq, + .outputs = wantedOutputs, + }); + + if (buildResult.success()) + amDone(ecSuccess); + else + amDone(ecFailed, Error("building '%s' failed", drvReq->to_string(worker.store))); +} + + +} diff --git a/src/libstore/build/create-derivation-and-realise-goal.hh b/src/libstore/build/create-derivation-and-realise-goal.hh new file mode 100644 index 000000000..ca936fc95 --- /dev/null +++ b/src/libstore/build/create-derivation-and-realise-goal.hh @@ -0,0 +1,96 @@ +#pragma once + +#include "parsed-derivations.hh" +#include "lock.hh" +#include "store-api.hh" +#include "pathlocks.hh" +#include "goal.hh" + +namespace nix { + +struct DerivationGoal; + +/** + * This goal type is essentially the serial composition (like function + * composition) of a goal for getting a derivation, and then a + * `DerivationGoal` using the newly-obtained derivation. + * + * In the (currently experimental) general inductive case of derivations + * that are themselves build outputs, that first goal will be *another* + * `CreateDerivationAndRealiseGoal`. In the (much more common) base-case + * where the derivation has no provence and is just referred to by + * (content-addressed) store path, that first goal is a + * `SubstitutionGoal`. + * + * If we already have the derivation (e.g. if the evalutator has created + * the derivation locally and then instructured the store to build it), + * we can skip the first goal entirely as a small optimization. + */ +struct CreateDerivationAndRealiseGoal : public Goal +{ + /** + * How to obtain a store path of the derivation to build. + */ + ref drvReq; + + /** + * The path of the derivation, once obtained. + **/ + std::optional optDrvPath; + + /** + * The goal for the corresponding concrete derivation. + **/ + std::shared_ptr concreteDrvGoal; + + /** + * The specific outputs that we need to build. + */ + OutputsSpec wantedOutputs; + + typedef void (CreateDerivationAndRealiseGoal::*GoalState)(); + GoalState state; + + /** + * The final output paths of the build. + * + * - For input-addressed derivations, always the precomputed paths + * + * - For content-addressed derivations, calcuated from whatever the + * hash ends up being. (Note that fixed outputs derivations that + * produce the "wrong" output still install that data under its + * true content-address.) + */ + OutputPathMap finalOutputs; + + BuildMode buildMode; + + CreateDerivationAndRealiseGoal(ref drvReq, + const OutputsSpec & wantedOutputs, Worker & worker, + BuildMode buildMode = bmNormal); + virtual ~CreateDerivationAndRealiseGoal(); + + void timedOut(Error && ex) override; + + std::string key() override; + + void work() override; + + /** + * Add wanted outputs to an already existing derivation goal. + */ + void addWantedOutputs(const OutputsSpec & outputs); + + /** + * The states. + */ + void getDerivation(); + void loadAndBuildDerivation(); + void buildDone(); + + JobCategory jobCategory() const override { + return JobCategory::Administration; + }; +}; + +} diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index ea56c0288..bec0bc538 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -71,7 +71,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, , wantedOutputs(wantedOutputs) , buildMode(buildMode) { - state = &DerivationGoal::getDerivation; + state = &DerivationGoal::loadDerivation; name = fmt( "building of '%s' from .drv file", DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store)); @@ -164,24 +164,6 @@ void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs) } -void DerivationGoal::getDerivation() -{ - trace("init"); - - /* The first thing to do is to make sure that the derivation - exists. If it doesn't, it may be created through a - substitute. */ - if (buildMode == bmNormal && worker.evalStore.isValidPath(drvPath)) { - loadDerivation(); - return; - } - - addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath))); - - state = &DerivationGoal::loadDerivation; -} - - void DerivationGoal::loadDerivation() { trace("loading derivation"); @@ -1493,7 +1475,7 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result) if (!useDerivation) return; auto & fullDrv = *dynamic_cast(drv.get()); - auto * dg = dynamic_cast(&*waitee); + auto * dg = tryGetConcreteDrvGoal(waitee); if (!dg) return; auto outputs = fullDrv.inputDrvs.find(dg->drvPath); diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 9d6fe1c0f..62b122c27 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -50,6 +50,13 @@ struct InitialOutput { std::optional known; }; +/** + * A goal for building some or all of the outputs of a derivation. + * + * The derivation must already be present, either in the store in a drv + * or in memory. If the derivation itself needs to be gotten first, a + * `CreateDerivationAndRealiseGoal` goal must be used instead. + */ struct DerivationGoal : public Goal { /** @@ -66,8 +73,7 @@ struct DerivationGoal : public Goal std::shared_ptr resolvedDrvGoal; /** - * The specific outputs that we need to build. Empty means all of - * them. + * The specific outputs that we need to build. */ OutputsSpec wantedOutputs; @@ -229,7 +235,6 @@ struct DerivationGoal : public Goal /** * The states. */ - void getDerivation(); void loadDerivation(); void haveDerivation(); void outputsSubstitutionTried(); @@ -334,7 +339,9 @@ struct DerivationGoal : public Goal StorePathSet exportReferences(const StorePathSet & storePaths); - JobCategory jobCategory() override { return JobCategory::Build; }; + JobCategory jobCategory() const override { + return JobCategory::Build; + }; }; MakeError(NotDeterministic, BuildError); diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh index 5d1253a71..da2426e5e 100644 --- a/src/libstore/build/drv-output-substitution-goal.hh +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -73,7 +73,9 @@ public: void work() override; void handleEOF(int fd) override; - JobCategory jobCategory() override { return JobCategory::Substitution; }; + JobCategory jobCategory() const override { + return JobCategory::Substitution; + }; }; } diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index f71fb35a6..f0f0e5519 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -1,5 +1,6 @@ #include "worker.hh" #include "substitution-goal.hh" +#include "create-derivation-and-realise-goal.hh" #include "derivation-goal.hh" #include "local-store.hh" @@ -15,7 +16,7 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod worker.run(goals); - StorePathSet failed; + StringSet failed; std::optional ex; for (auto & i : goals) { if (i->ex) { @@ -25,8 +26,10 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod ex = std::move(i->ex); } if (i->exitCode != Goal::ecSuccess) { - if (auto i2 = dynamic_cast(i.get())) failed.insert(i2->drvPath); - else if (auto i2 = dynamic_cast(i.get())) failed.insert(i2->storePath); + if (auto i2 = dynamic_cast(i.get())) + failed.insert(i2->drvReq->to_string(*this)); + else if (auto i2 = dynamic_cast(i.get())) + failed.insert(printStorePath(i2->storePath)); } } @@ -35,7 +38,7 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod throw std::move(*ex); } else if (!failed.empty()) { if (ex) logError(ex->info()); - throw Error(worker.failingExitStatus(), "build of %s failed", showPaths(failed)); + throw Error(worker.failingExitStatus(), "build of %s failed", concatStringsSep(", ", quoteStrings(failed))); } } diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index ca7097a68..f8db98280 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -11,7 +11,7 @@ bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const { } -BuildResult Goal::getBuildResult(const DerivedPath & req) { +BuildResult Goal::getBuildResult(const DerivedPath & req) const { BuildResult res { buildResult }; if (auto pbp = std::get_if(&req)) { diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index d3127caea..01d3c3491 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -41,8 +41,24 @@ typedef std::map WeakGoalMap; * of each category in parallel. */ enum struct JobCategory { + /** + * A build of a derivation; it will use CPU and disk resources. + */ Build, + /** + * A substitution an arbitrary store object; it will use network resources. + */ Substitution, + /** + * A goal that does no "real" work by itself, and just exists to depend on + * other goals which *do* do real work. These goals therefore are not + * limited. + * + * These goals cannot infinitely create themselves, so there is no risk of + * a "fork bomb" type situation (which would be a problem even though the + * goal do no real work) either. + */ + Administration, }; struct Goal : public std::enable_shared_from_this @@ -110,7 +126,7 @@ public: * sake of both privacy and determinism, and this "safe accessor" * ensures we don't. */ - BuildResult getBuildResult(const DerivedPath &); + BuildResult getBuildResult(const DerivedPath &) const; /** * Exception containing an error message, if any. @@ -144,7 +160,7 @@ public: void trace(std::string_view s); - std::string getName() + std::string getName() const { return name; } @@ -166,7 +182,7 @@ public: * @brief Hint for the scheduler, which concurrency limit applies. * @see JobCategory */ - virtual JobCategory jobCategory() = 0; + virtual JobCategory jobCategory() const = 0; }; void addToWeakGoals(WeakGoals & goals, GoalPtr p); diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index 1b693baa1..1d389d328 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -117,7 +117,9 @@ public: /* Called by destructor, can't be overridden */ void cleanup() override final; - JobCategory jobCategory() override { return JobCategory::Substitution; }; + JobCategory jobCategory() const override { + return JobCategory::Substitution; + }; }; } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index b58fc5c1c..f65f63b99 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -2,6 +2,7 @@ #include "worker.hh" #include "substitution-goal.hh" #include "drv-output-substitution-goal.hh" +#include "create-derivation-and-realise-goal.hh" #include "local-derivation-goal.hh" #include "hook-instance.hh" @@ -41,6 +42,24 @@ Worker::~Worker() } +std::shared_ptr Worker::makeCreateDerivationAndRealiseGoal( + ref drvReq, + const OutputsSpec & wantedOutputs, + BuildMode buildMode) +{ + std::weak_ptr & goal_weak = outerDerivationGoals.ensureSlot(*drvReq).value; + std::shared_ptr goal = goal_weak.lock(); + if (!goal) { + goal = std::make_shared(drvReq, wantedOutputs, *this, buildMode); + goal_weak = goal; + wakeUp(goal); + } else { + goal->addWantedOutputs(wantedOutputs); + } + return goal; +} + + std::shared_ptr Worker::makeDerivationGoalCommon( const StorePath & drvPath, const OutputsSpec & wantedOutputs, @@ -111,10 +130,7 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) { return std::visit(overloaded { [&](const DerivedPath::Built & bfd) -> GoalPtr { - if (auto bop = std::get_if(&*bfd.drvPath)) - return makeDerivationGoal(bop->path, bfd.outputs, buildMode); - else - throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented."); + return makeCreateDerivationAndRealiseGoal(bfd.drvPath, bfd.outputs, buildMode); }, [&](const DerivedPath::Opaque & bo) -> GoalPtr { return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair); @@ -123,24 +139,46 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) } +template +static void cullMap(std::map & goalMap, F f) +{ + for (auto i = goalMap.begin(); i != goalMap.end();) + if (!f(i->second)) + i = goalMap.erase(i); + else ++i; +} + + template static void removeGoal(std::shared_ptr goal, std::map> & goalMap) { /* !!! inefficient */ - for (auto i = goalMap.begin(); - i != goalMap.end(); ) - if (i->second.lock() == goal) { - auto j = i; ++j; - goalMap.erase(i); - i = j; - } - else ++i; + cullMap(goalMap, [&](const std::weak_ptr & gp) -> bool { + return gp.lock() != goal; + }); +} + +template +static void removeGoal(std::shared_ptr goal, std::map>::ChildNode> & goalMap); + +template +static void removeGoal(std::shared_ptr goal, std::map>::ChildNode> & goalMap) +{ + /* !!! inefficient */ + cullMap(goalMap, [&](DerivedPathMap>::ChildNode & node) -> bool { + if (node.value.lock() == goal) + node.value.reset(); + removeGoal(goal, node.childMap); + return !node.value.expired() || !node.childMap.empty(); + }); } void Worker::removeGoal(GoalPtr goal) { - if (auto drvGoal = std::dynamic_pointer_cast(goal)) + if (auto drvGoal = std::dynamic_pointer_cast(goal)) + nix::removeGoal(drvGoal, outerDerivationGoals.map); + else if (auto drvGoal = std::dynamic_pointer_cast(goal)) nix::removeGoal(drvGoal, derivationGoals); else if (auto subGoal = std::dynamic_pointer_cast(goal)) nix::removeGoal(subGoal, substitutionGoals); @@ -198,8 +236,19 @@ void Worker::childStarted(GoalPtr goal, const std::set & fds, child.respectTimeouts = respectTimeouts; children.emplace_back(child); if (inBuildSlot) { - if (goal->jobCategory() == JobCategory::Substitution) nrSubstitutions++; - else nrLocalBuilds++; + switch (goal->jobCategory()) { + case JobCategory::Substitution: + nrSubstitutions++; + break; + case JobCategory::Build: + nrLocalBuilds++; + break; + case JobCategory::Administration: + /* Intentionally not limited, see docs */ + break; + default: + abort(); + } } } @@ -211,12 +260,20 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers) if (i == children.end()) return; if (i->inBuildSlot) { - if (goal->jobCategory() == JobCategory::Substitution) { + switch (goal->jobCategory()) { + case JobCategory::Substitution: assert(nrSubstitutions > 0); nrSubstitutions--; - } else { + break; + case JobCategory::Build: assert(nrLocalBuilds > 0); nrLocalBuilds--; + break; + case JobCategory::Administration: + /* Intentionally not limited, see docs */ + break; + default: + abort(); } } @@ -267,9 +324,9 @@ void Worker::run(const Goals & _topGoals) for (auto & i : _topGoals) { topGoals.insert(i); - if (auto goal = dynamic_cast(i.get())) { + if (auto goal = dynamic_cast(i.get())) { topPaths.push_back(DerivedPath::Built { - .drvPath = makeConstantStorePathRef(goal->drvPath), + .drvPath = goal->drvReq, .outputs = goal->wantedOutputs, }); } else if (auto goal = dynamic_cast(i.get())) { @@ -522,11 +579,26 @@ void Worker::markContentsGood(const StorePath & path) } -GoalPtr upcast_goal(std::shared_ptr subGoal) { - return subGoal; -} -GoalPtr upcast_goal(std::shared_ptr subGoal) { +GoalPtr upcast_goal(std::shared_ptr subGoal) +{ return subGoal; } +GoalPtr upcast_goal(std::shared_ptr subGoal) +{ + return subGoal; +} + +GoalPtr upcast_goal(std::shared_ptr subGoal) +{ + return subGoal; +} + +const DerivationGoal * tryGetConcreteDrvGoal(GoalPtr waitee) +{ + auto * odg = dynamic_cast(&*waitee); + if (!odg) return nullptr; + return &*odg->concreteDrvGoal; +} + } diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 5abceca0d..a778e311c 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -4,6 +4,7 @@ #include "types.hh" #include "lock.hh" #include "store-api.hh" +#include "derived-path-map.hh" #include "goal.hh" #include "realisation.hh" @@ -13,6 +14,7 @@ namespace nix { /* Forward definition. */ +struct CreateDerivationAndRealiseGoal; struct DerivationGoal; struct PathSubstitutionGoal; class DrvOutputSubstitutionGoal; @@ -31,9 +33,23 @@ class DrvOutputSubstitutionGoal; */ GoalPtr upcast_goal(std::shared_ptr subGoal); GoalPtr upcast_goal(std::shared_ptr subGoal); +GoalPtr upcast_goal(std::shared_ptr subGoal); typedef std::chrono::time_point steady_time_point; +/** + * The current implementation of impure derivations has + * `DerivationGoal`s accumulate realisations from their waitees. + * Unfortunately, `DerivationGoal`s don't directly depend on other + * goals, but instead depend on `CreateDerivationAndRealiseGoal`s. + * + * We try not to share any of the details of any goal type with any + * other, for sake of modularity and quicker rebuilds. This means we + * cannot "just" downcast and fish out the field. So as an escape hatch, + * we have made the function, written in `worker.cc` where all the goal + * types are visible, and use it instead. + */ +const DerivationGoal * tryGetConcreteDrvGoal(GoalPtr waitee); /** * A mapping used to remember for each child process to what goal it @@ -102,6 +118,9 @@ private: * Maps used to prevent multiple instantiations of a goal for the * same derivation / path. */ + + DerivedPathMap> outerDerivationGoals; + std::map> derivationGoals; std::map> substitutionGoals; std::map> drvOutputSubstitutionGoals; @@ -189,6 +208,9 @@ public: * @ref DerivationGoal "derivation goal" */ private: + std::shared_ptr makeCreateDerivationAndRealiseGoal( + ref drvPath, + const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal); std::shared_ptr makeDerivationGoalCommon( const StorePath & drvPath, const OutputsSpec & wantedOutputs, std::function()> mkDrvGoal); diff --git a/src/libstore/derived-path-map.cc b/src/libstore/derived-path-map.cc new file mode 100644 index 000000000..5c8c7a4f2 --- /dev/null +++ b/src/libstore/derived-path-map.cc @@ -0,0 +1,33 @@ +#include "derived-path-map.hh" + +namespace nix { + +template +typename DerivedPathMap::ChildNode & DerivedPathMap::ensureSlot(const SingleDerivedPath & k) +{ + std::function initIter; + initIter = [&](const auto & k) -> auto & { + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) -> auto & { + // will not overwrite if already there + return map[bo.path]; + }, + [&](const SingleDerivedPath::Built & bfd) -> auto & { + auto & n = initIter(*bfd.drvPath); + return n.childMap[bfd.output]; + }, + }, k.raw()); + }; + return initIter(k); +} + +} + +// instantiations + +#include "create-derivation-and-realise-goal.hh" +namespace nix { + +template struct DerivedPathMap>; + +} diff --git a/src/libstore/derived-path-map.hh b/src/libstore/derived-path-map.hh new file mode 100644 index 000000000..9ce58206e --- /dev/null +++ b/src/libstore/derived-path-map.hh @@ -0,0 +1,73 @@ +#pragma once + +#include "types.hh" +#include "derived-path.hh" + +namespace nix { + +/** + * A simple Trie, of sorts. Conceptually a map of `SingleDerivedPath` to + * values. + * + * Concretely, an n-ary tree, as described below. A + * `SingleDerivedPath::Opaque` maps to the value of an immediate child + * of the root node. A `SingleDerivedPath::Built` maps to a deeper child + * node: the `SingleDerivedPath::Built::drvPath` is first mapped to a a + * child node (inductively), and then the + * `SingleDerivedPath::Built::output` is used to look up that child's + * child via its map. In this manner, every `SingleDerivedPath` is + * mapped to a child node. + * + * @param V A type to instantiate for each output. It should probably + * should be an "optional" type so not every interior node has to have a + * value. For example, the scheduler uses + * `DerivedPathMap>` to + * remember which goals correspond to which outputs. `* const Something` + * or `std::optional` would also be good choices for + * "optional" types. + */ +template +struct DerivedPathMap { + /** + * A child node (non-root node). + */ + struct ChildNode { + /** + * Value of this child node. + * + * @see DerivedPathMap for what `V` should be. + */ + V value; + + /** + * The map type for the root node. + */ + using Map = std::map; + + /** + * The map of the root node. + */ + Map childMap; + }; + + /** + * The map type for the root node. + */ + using Map = std::map; + + /** + * The map of root node. + */ + Map map; + + /** + * Find the node for `k`, creating it if needed. + * + * The node is referred to as a "slot" on the assumption that `V` is + * some sort of optional type, so the given key can be set or unset + * by changing this node. + */ + ChildNode & ensureSlot(const SingleDerivedPath & k); +}; + +} diff --git a/tests/dyn-drv/build-built-drv.sh b/tests/dyn-drv/build-built-drv.sh index 647be9457..94f3550bd 100644 --- a/tests/dyn-drv/build-built-drv.sh +++ b/tests/dyn-drv/build-built-drv.sh @@ -18,4 +18,6 @@ clearStore drvDep=$(nix-instantiate ./text-hashed-output.nix -A producingDrv) -expectStderr 1 nix build "${drvDep}^out^out" --no-link | grepQuiet "Building dynamic derivations in one shot is not yet implemented" +out2=$(nix build "${drvDep}^out^out" --no-link) + +test $out1 == $out2 From d2e6cfa0750cb38ceef7dc0b8bb31cf3b0387e9c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 25 Aug 2023 17:17:33 +0200 Subject: [PATCH 027/139] tests/lang/eval-okay-pathexists: Add cases --- tests/lang/eval-okay-pathexists.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/lang/eval-okay-pathexists.nix b/tests/lang/eval-okay-pathexists.nix index 50c28ee0c..8eae37e70 100644 --- a/tests/lang/eval-okay-pathexists.nix +++ b/tests/lang/eval-okay-pathexists.nix @@ -1,4 +1,6 @@ -builtins.pathExists (builtins.toPath ./lib.nix) +builtins.pathExists (./lib.nix) +&& builtins.pathExists (builtins.toPath ./lib.nix) +&& builtins.pathExists (builtins.toString ./lib.nix) && builtins.pathExists (builtins.toPath (builtins.toString ./lib.nix)) && !builtins.pathExists (builtins.toPath (builtins.toString ./bla.nix)) && builtins.pathExists ./lib.nix From 1e08e12d8138b09e6872cb498b723ade9ad71d68 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 25 Aug 2023 17:18:37 +0200 Subject: [PATCH 028/139] pathExists: isDir when endswith / Fixes https://github.com/NixOS/nix/issues/8838 --- src/libexpr/primops.cc | 16 +++++++++++++--- tests/lang/eval-okay-pathexists.nix | 1 + 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 283f99a48..915b872c8 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1520,15 +1520,25 @@ static RegisterPrimOp primop_storePath({ static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args, Value & v) { + auto & arg = *args[0]; + /* We don’t check the path right now, because we don’t want to throw if the path isn’t allowed, but just return false (and we can’t just catch the exception here because we still want to - throw if something in the evaluation of `*args[0]` tries to + throw if something in the evaluation of `arg` tries to access an unauthorized path). */ - auto path = realisePath(state, pos, *args[0], { .checkForPureEval = false }); + auto path = realisePath(state, pos, arg, { .checkForPureEval = false }); + + /* SourcePath doesn't know about trailing slash. */ + auto mustBeDir = arg.type() == nString && arg.str().ends_with("/"); try { - v.mkBool(state.checkSourcePath(path).pathExists()); + auto checked = state.checkSourcePath(path); + auto exists = checked.pathExists(); + if (exists && mustBeDir) { + exists = checked.lstat().type == InputAccessor::tDirectory; + } + v.mkBool(exists); } catch (SysError & e) { /* Don't give away info from errors while canonicalising ‘path’ in restricted mode. */ diff --git a/tests/lang/eval-okay-pathexists.nix b/tests/lang/eval-okay-pathexists.nix index 8eae37e70..e1246e370 100644 --- a/tests/lang/eval-okay-pathexists.nix +++ b/tests/lang/eval-okay-pathexists.nix @@ -1,6 +1,7 @@ builtins.pathExists (./lib.nix) && builtins.pathExists (builtins.toPath ./lib.nix) && builtins.pathExists (builtins.toString ./lib.nix) +&& !builtins.pathExists (builtins.toString ./lib.nix + "/") && builtins.pathExists (builtins.toPath (builtins.toString ./lib.nix)) && !builtins.pathExists (builtins.toPath (builtins.toString ./bla.nix)) && builtins.pathExists ./lib.nix From 736b9cede73692a1cf92a6c21c5259498a04c961 Mon Sep 17 00:00:00 2001 From: Bryan Honof Date: Mon, 28 Aug 2023 15:43:34 +0200 Subject: [PATCH 029/139] Port the flags of nix-daemon to nix daemon (#8788) The new `nix daemon` command didn't accept the same flags that `nix-daemon` did. * docs(daemon): clarify the daemon trust override flags * fix: change declaration order * docs: add examples of nix daemon usage * Apply suggestions from code review --------- Co-authored-by: Eelco Dolstra Co-authored-by: John Ericson Co-authored-by: tomberek --- src/nix/daemon.cc | 41 ++++++++++++++++++++++++++++++++++++++++- src/nix/daemon.md | 30 +++++++++++++++++++++++++++--- 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 1511f9e6e..af428018a 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -500,6 +500,45 @@ static RegisterLegacyCommand r_nix_daemon("nix-daemon", main_nix_daemon); struct CmdDaemon : StoreCommand { + bool stdio = false; + std::optional isTrustedOpt = std::nullopt; + + CmdDaemon() + { + addFlag({ + .longName = "stdio", + .description = "Attach to standard I/O, instead of trying to bind to a UNIX socket.", + .handler = {&stdio, true}, + }); + + addFlag({ + .longName = "force-trusted", + .description = "Force the daemon to trust connecting clients.", + .handler = {[&]() { + isTrustedOpt = Trusted; + }}, + .experimentalFeature = Xp::DaemonTrustOverride, + }); + + addFlag({ + .longName = "force-untrusted", + .description = "Force the daemon to not trust connecting clients. The connection will be processed by the receiving daemon before forwarding commands.", + .handler = {[&]() { + isTrustedOpt = NotTrusted; + }}, + .experimentalFeature = Xp::DaemonTrustOverride, + }); + + addFlag({ + .longName = "default-trust", + .description = "Use Nix's default trust.", + .handler = {[&]() { + isTrustedOpt = std::nullopt; + }}, + .experimentalFeature = Xp::DaemonTrustOverride, + }); + } + std::string description() override { return "daemon to perform store operations on behalf of non-root clients"; @@ -516,7 +555,7 @@ struct CmdDaemon : StoreCommand void run(ref store) override { - runDaemon(false, std::nullopt); + runDaemon(stdio, isTrustedOpt); } }; diff --git a/src/nix/daemon.md b/src/nix/daemon.md index d5cdadf08..b1ea850ed 100644 --- a/src/nix/daemon.md +++ b/src/nix/daemon.md @@ -1,20 +1,44 @@ R""( -# Example +# Examples -* Run the daemon in the foreground: +* Run the daemon: ```console # nix daemon ``` +* Run the daemon and listen on standard I/O instead of binding to a UNIX socket: + + ```console + # nix daemon --stdio + ``` + +* Run the daemon and force all connections to be trusted: + + ```console + # nix daemon --force-trusted + ``` + +* Run the daemon and force all connections to be untrusted: + + ```console + # nix daemon --force-untrusted + ``` + +* Run the daemon, listen on standard I/O, and force all connections to use Nix's default trust: + + ```console + # nix daemon --stdio --default-trust + ``` + # Description This command runs the Nix daemon, which is a required component in multi-user Nix installations. It runs build tasks and other operations on the Nix store on behalf of non-root users. Usually you don't run the daemon directly; instead it's managed by a service -management framework such as `systemd`. +management framework such as `systemd` on Linux, or `launchctl` on Darwin. Note that this daemon does not fork into the background. From 880fef9cdf3b732fa76ec47710edaf572bf91e92 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 28 Aug 2023 20:51:44 +0200 Subject: [PATCH 030/139] do not change existing release notes --- doc/manual/src/release-notes/rl-2.12.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/release-notes/rl-2.12.md b/doc/manual/src/release-notes/rl-2.12.md index 57d092e01..e1e3efe1a 100644 --- a/doc/manual/src/release-notes/rl-2.12.md +++ b/doc/manual/src/release-notes/rl-2.12.md @@ -2,8 +2,19 @@ * On Linux, Nix can now run builds in a user namespace where they run as root (UID 0) and have 65,536 UIDs available. + This is primarily useful for running containers such as `systemd-nspawn` + inside a Nix build. For an example, see [`tests/systemd-nspawn/nix`][nspawn]. - This can be used by requiring `uid-range` [system feature] in derivations. + [nspawn]: https://github.com/NixOS/nix/blob/67bcb99700a0da1395fa063d7c6586740b304598/tests/systemd-nspawn.nix. + + A build can enable this by setting the derivation attribute: + + ``` + requiredSystemFeatures = [ "uid-range" ]; + ``` + + The `uid-range` [system feature] requires the [`auto-allocate-uids`] + setting to be enabled. [system feature]: ../command-ref/conf-file.md#conf-system-features From 151120a1ae082c6460f9a54cf795c57d154f6c27 Mon Sep 17 00:00:00 2001 From: Silvan Mosberger Date: Mon, 28 Aug 2023 22:14:01 +0200 Subject: [PATCH 031/139] Document nix-prefetch-url defaults (#8878) --- doc/manual/src/command-ref/nix-prefetch-url.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/doc/manual/src/command-ref/nix-prefetch-url.md b/doc/manual/src/command-ref/nix-prefetch-url.md index 3bcd209e2..45ef01e02 100644 --- a/doc/manual/src/command-ref/nix-prefetch-url.md +++ b/doc/manual/src/command-ref/nix-prefetch-url.md @@ -31,15 +31,18 @@ store already contains a file with the same hash and base name. Otherwise, the file is downloaded, and an error is signaled if the actual hash of the file does not match the specified hash. -This command prints the hash on standard output. Additionally, if the -option `--print-path` is used, the path of the downloaded file in the -Nix store is also printed. +This command prints the hash on standard output. +The hash is printed using base-32 unless `--type md5` is specified, +in which case it's printed using base-16. +Additionally, if the option `--print-path` is used, +the path of the downloaded file in the Nix store is also printed. # Options - `--type` *hashAlgo*\ - Use the specified cryptographic hash algorithm, which can be one of - `md5`, `sha1`, `sha256`, and `sha512`. + Use the specified cryptographic hash algorithm, + which can be one of `md5`, `sha1`, `sha256`, and `sha512`. + The default is `sha256`. - `--print-path`\ Print the store path of the downloaded file on standard output. From 56763ff918eb308db23080e560ed2ea3e00c80a7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 29 Aug 2023 15:04:06 +0200 Subject: [PATCH 032/139] Document that redirected tarball flakerefs can specify lastModified --- doc/manual/src/protocols/tarball-fetcher.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/protocols/tarball-fetcher.md b/doc/manual/src/protocols/tarball-fetcher.md index 0d3212303..274fa6d63 100644 --- a/doc/manual/src/protocols/tarball-fetcher.md +++ b/doc/manual/src/protocols/tarball-fetcher.md @@ -20,8 +20,8 @@ Link: ; rel="immutable" (Note the required `<` and `>` characters around *flakeref*.) -*flakeref* must be a tarball flakeref. It can contain flake attributes -such as `narHash`, `rev` and `revCount`. If `narHash` is included, its +*flakeref* must be a tarball flakeref. It can contain the tarball flake attributes +`narHash`, `rev`, `revCount` and `lastModified`. If `narHash` is included, its value must be the NAR hash of the unpacked tarball (as computed via `nix hash path`). Nix checks the contents of the returned tarball against the `narHash` attribute. The `rev` and `revCount` attributes From 46478b44ffb477387235b3c597c24178845fb2a3 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Tue, 29 Aug 2023 15:13:35 -0700 Subject: [PATCH 033/139] docs/testing: point out the existence of `GTEST_FILTER` (#8883) --- doc/manual/src/contributing/testing.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index c3c82e3c0..cd94d5cfb 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -7,7 +7,8 @@ under `src/{library_name}/tests` using the [googletest](https://google.github.io/googletest/) and [rapidcheck](https://github.com/emil-e/rapidcheck) frameworks. -You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option. +You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. +Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option, or the `GTEST_FILTER` environment variable. ## Functional tests From 3384f70a3d5ee7bd1d1bf5fa48809c4eac5c8041 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Fri, 11 Aug 2023 15:58:43 +0200 Subject: [PATCH 034/139] nixpkgsLibTests: Only test our Nix Interface has changed upstream. It *should* be fine to test 23.05's other Nix versions as those *should* succeed, but that's not the case and it's obfuscating our terrible CI setup's log. --- flake.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index cf7c1fa12..36ba61484 100644 --- a/flake.nix +++ b/flake.nix @@ -656,7 +656,9 @@ tests.nixpkgsLibTests = forAllSystems (system: import (nixpkgs + "/lib/tests/release.nix") - { pkgs = nixpkgsFor.${system}.native; } + { pkgs = nixpkgsFor.${system}.native; + nixVersions = [ self.packages.${system}.nix ]; + } ); metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" { From be3362e74782e47b9546de94be97f53540ddfe9e Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 14 Aug 2023 14:28:02 +0200 Subject: [PATCH 035/139] Fix nix-copy test --- tests/nixos/nix-copy.nix | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/nixos/nix-copy.nix b/tests/nixos/nix-copy.nix index ef053de03..2981cc2b8 100644 --- a/tests/nixos/nix-copy.nix +++ b/tests/nixos/nix-copy.nix @@ -1,4 +1,6 @@ # Test that ‘nix copy’ works over ssh. +# Run interactively with: +# rm key key.pub; nix run .#hydraJobs.tests.nix-copy.driverInteractive { lib, config, nixpkgs, hostPkgs, ... }: @@ -55,7 +57,9 @@ in { server.wait_for_unit("sshd") client.wait_for_unit("network.target") client.wait_for_unit("getty@tty1.service") - client.wait_for_text("]#") + # Either the prompt: ]# + # or an OCR misreading of it: 1# + client.wait_for_text("[]1]#") # Copy the closure of package A from the client to the server using password authentication, # and check that all prompts are visible From 1bc9257d7c5980fb048b519eeaa2a2b2e8925099 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 3 May 2023 15:36:13 +0200 Subject: [PATCH 036/139] reword description of how realisation works --- .../src/command-ref/nix-store/opt-common.md | 4 +- .../src/command-ref/nix-store/realise.md | 50 +++++++++---------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/doc/manual/src/command-ref/nix-store/opt-common.md b/doc/manual/src/command-ref/nix-store/opt-common.md index dd9a6bf21..104b20076 100644 --- a/doc/manual/src/command-ref/nix-store/opt-common.md +++ b/doc/manual/src/command-ref/nix-store/opt-common.md @@ -1,6 +1,6 @@ -# Options +# Command options -The following options are allowed for all `nix-store` operations, but may not always have an effect. +The following options are allowed for all operations in the `nix-store` command, but may not always have an effect. - [`--add-root`](#opt-add-root) *path* diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index c19aea75e..2881a69b3 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -8,33 +8,37 @@ # Description -The operation `--realise` essentially “builds” the specified store -paths. Realisation is a somewhat overloaded term: +Ensure that the given [store paths] are [valid]. - - If the store path is a *derivation*, realisation ensures that the - output paths of the derivation are [valid] (i.e., - the output path and its closure exist in the file system). This - can be done in several ways. First, it is possible that the - outputs are already valid, in which case we are done - immediately. Otherwise, there may be [substitutes] - that produce the outputs (e.g., by downloading them). Finally, the - outputs can be produced by running the build task described - by the derivation. +Realisation of a store path works as follows: - - If the store path is not a derivation, realisation ensures that the - specified path is valid (i.e., it and its closure exist in the file - system). If the path is already valid, we are done immediately. - Otherwise, the path and any missing paths in its closure may be - produced through substitutes. If there are no (successful) - substitutes, realisation fails. +- If the path is already valid, do nothing. +- If the path leads to a [store derivation]: + 1. Realise the store derivation file itself, as a regular store path. + 2. Realise its [output paths]: + - Try to fetch from [substituters] the [store objects] associated with the output paths in the store derivation's [closure]. + - With [content-addressed derivations] (experimental): Determine the output paths to realise by querying build certificates in the [Nix database]. + - For any store paths that cannot be substituted, produce the required store objects by first realising all outputs of the derivation's dependencies and then running the derivation's build instructions. +- Otherwise: Try to fetch the associated [store objects] in the paths' [closure] from the [substituters]. +If no substitutes are available and no store derivation is given, realisation fails. + +[store paths]: @docroot@/glossary.md#gloss-store-path [valid]: @docroot@/glossary.md#gloss-validity -[substitutes]: @docroot@/glossary.md#gloss-substitute +[store derivation]: @docroot@/glossary.md#gloss-store-derivation +[output paths]: @docroot@/glossary.md#gloss-output-path +[store objects]: @docroot@/glossary.md#gloss-store-object +[closure]: @docroot@/glossary.md#gloss-closure +[substituters]: @docroot@/command-ref/conf-file.md#conf-substituters +[content-addressed derivations]: @docroot@/contributing/experimental-features.md#xp-feature-ca-derivations +[Nix database]: @docroot@/glossary.md#gloss-nix-database -The output path of each derivation is printed on standard output. (For -non-derivations argument, the argument itself is printed.) +The resulting paths are printed on standard output. +For non-derivation arguments, the argument itself is printed. -The following flags are available: +{{#include ../status-build-failure.md}} + +# Options - `--dry-run`\ Print on standard error a description of what packages would be @@ -54,8 +58,6 @@ The following flags are available: previous build, the new output path is left in `/nix/store/name.check.` -{{#include ../status-build-failure.md}} - {{#include ./opt-common.md}} {{#include ../opt-common.md}} @@ -67,8 +69,6 @@ The following flags are available: This operation is typically used to build [store derivation]s produced by [`nix-instantiate`](@docroot@/command-ref/nix-instantiate.md): -[store derivation]: @docroot@/glossary.md#gloss-store-derivation - ```console $ nix-store --realise $(nix-instantiate ./test.nix) /nix/store/31axcgrlbfsxzmfff1gyj1bf62hvkby2-aterm-2.3.1 From 315a11bcc9ce97cb25a41dcdfb4a859c7384f74b Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 21 Jun 2023 10:54:46 +0200 Subject: [PATCH 037/139] remove superfluous word --- doc/manual/src/command-ref/nix-store/realise.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index 2881a69b3..f304dde2e 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -19,7 +19,7 @@ Realisation of a store path works as follows: - Try to fetch from [substituters] the [store objects] associated with the output paths in the store derivation's [closure]. - With [content-addressed derivations] (experimental): Determine the output paths to realise by querying build certificates in the [Nix database]. - For any store paths that cannot be substituted, produce the required store objects by first realising all outputs of the derivation's dependencies and then running the derivation's build instructions. -- Otherwise: Try to fetch the associated [store objects] in the paths' [closure] from the [substituters]. +- Otherwise: Try to fetch the associated [store objects] in the paths' [closure] from [substituters]. If no substitutes are available and no store derivation is given, realisation fails. From a57e0e8c5c1be81e9352499ac31dcfce87ca44be Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 27 Jun 2023 09:16:36 +0200 Subject: [PATCH 038/139] reword introductory sentence --- doc/manual/src/command-ref/nix-store/realise.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index f304dde2e..00acee99c 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -10,7 +10,7 @@ Ensure that the given [store paths] are [valid]. -Realisation of a store path works as follows: +Each of *paths* is processed as follows: - If the path is already valid, do nothing. - If the path leads to a [store derivation]: From b7e9e2960566b8e21f4f69b6f222b945fababce7 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 27 Jun 2023 09:16:56 +0200 Subject: [PATCH 039/139] remove abstract description --- doc/manual/src/command-ref/nix-store/realise.md | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index 00acee99c..72dfa618c 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -8,7 +8,6 @@ # Description -Ensure that the given [store paths] are [valid]. Each of *paths* is processed as follows: From d50f116421f7c999748887955b2d3c2e8b934b28 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 27 Jun 2023 09:17:12 +0200 Subject: [PATCH 040/139] add reference link --- doc/manual/src/command-ref/nix-store/realise.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index 72dfa618c..2028631d0 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -11,7 +11,7 @@ Each of *paths* is processed as follows: -- If the path is already valid, do nothing. +- If the path is already [valid], do nothing. - If the path leads to a [store derivation]: 1. Realise the store derivation file itself, as a regular store path. 2. Realise its [output paths]: From 0cd8f36644042b41516e86e527ebfb19faebcbc9 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 13 Jul 2023 22:17:51 +0200 Subject: [PATCH 041/139] add anchor to `builder` --- doc/manual/src/command-ref/nix-store/realise.md | 2 +- doc/manual/src/glossary.md | 2 +- doc/manual/src/language/derivations.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index 2028631d0..9dfe74277 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -17,7 +17,7 @@ Each of *paths* is processed as follows: 2. Realise its [output paths]: - Try to fetch from [substituters] the [store objects] associated with the output paths in the store derivation's [closure]. - With [content-addressed derivations] (experimental): Determine the output paths to realise by querying build certificates in the [Nix database]. - - For any store paths that cannot be substituted, produce the required store objects by first realising all outputs of the derivation's dependencies and then running the derivation's build instructions. + - For any store paths that cannot be substituted, produce the required store objects by first realising all outputs of the derivation's dependencies and then running the derivation's [`builder`](@docroot@/language/derivations.md#attr-builder) executable. - Otherwise: Try to fetch the associated [store objects] in the paths' [closure] from [substituters]. If no substitutes are available and no store derivation is given, realisation fails. diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index d950a4ca8..2fbb23311 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -33,7 +33,7 @@ Ensure a [store path] is [valid][validity]. - This means either running the `builder` executable as specified in the corresponding [derivation] or fetching a pre-built [store object] from a [substituter]. + This means either running the [`builder`](@docroot@/language/derivations.md#attr-builder) executable as specified in the corresponding [derivation] or fetching a pre-built [store object] from a [substituter]. See [`nix-build`](./command-ref/nix-build.md) and [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md). diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index 043a38191..79a09122a 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -17,7 +17,7 @@ the attributes of which specify the inputs of the build. string. This is used as a symbolic name for the package by `nix-env`, and it is appended to the output paths of the derivation. - - There must be an attribute named `builder` that identifies the + - There must be an attribute named [`builder`]{#attr-builder} that identifies the program that is executed to perform the build. It can be either a derivation or a source (a local file reference, e.g., `./builder.sh`). From 6b3320ab05cefe73c21a469d077b277cff47733b Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 19 Jul 2023 09:27:36 +0200 Subject: [PATCH 042/139] mention remote builders Co-authored-by: Robert Hensing --- doc/manual/src/glossary.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 2fbb23311..8f0b25c89 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -33,7 +33,7 @@ Ensure a [store path] is [valid][validity]. - This means either running the [`builder`](@docroot@/language/derivations.md#attr-builder) executable as specified in the corresponding [derivation] or fetching a pre-built [store object] from a [substituter]. + This means either running the [`builder`](@docroot@/language/derivations.md#attr-builder) executable as specified in the corresponding [derivation], or fetching a pre-built [store object] from a [substituter], or delegating to a [remote builder](@docroot@/advanced-topics/distributed-builds.html) and retrieving the outputs. See [`nix-build`](./command-ref/nix-build.md) and [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md). From d460dbdd30e3b518de962314bcf183961caf7f53 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 19 Jul 2023 09:31:03 +0200 Subject: [PATCH 043/139] be more precise about substituting store derivations Co-authored-by: Robert Hensing --- doc/manual/src/command-ref/nix-store/realise.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index 9dfe74277..750cbeaff 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -11,9 +11,8 @@ Each of *paths* is processed as follows: -- If the path is already [valid], do nothing. - If the path leads to a [store derivation]: - 1. Realise the store derivation file itself, as a regular store path. + 1. If it is not [valid], substitute the store derivation file itself. 2. Realise its [output paths]: - Try to fetch from [substituters] the [store objects] associated with the output paths in the store derivation's [closure]. - With [content-addressed derivations] (experimental): Determine the output paths to realise by querying build certificates in the [Nix database]. From cf4e14d58da04465c2afd05af6975d568f58ce1d Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 19 Jul 2023 13:28:44 +0200 Subject: [PATCH 044/139] accommodate "do nothing" branch --- doc/manual/src/command-ref/nix-store/realise.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index 750cbeaff..6dc908c19 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -17,7 +17,7 @@ Each of *paths* is processed as follows: - Try to fetch from [substituters] the [store objects] associated with the output paths in the store derivation's [closure]. - With [content-addressed derivations] (experimental): Determine the output paths to realise by querying build certificates in the [Nix database]. - For any store paths that cannot be substituted, produce the required store objects by first realising all outputs of the derivation's dependencies and then running the derivation's [`builder`](@docroot@/language/derivations.md#attr-builder) executable. -- Otherwise: Try to fetch the associated [store objects] in the paths' [closure] from [substituters]. +- Otherwise, and if the path is not already valid: Try to fetch the associated [store objects] in the path's [closure] from [substituters]. If no substitutes are available and no store derivation is given, realisation fails. From b951e862d0bdb93a64e2c85af8762a9294d575c9 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 19 Jul 2023 13:30:40 +0200 Subject: [PATCH 045/139] more meaningful tagline --- doc/manual/src/command-ref/nix-store/realise.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index 6dc908c19..680e54ffa 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -1,6 +1,6 @@ # Name -`nix-store --realise` - realise specified store paths +`nix-store --realise` - build or fetch store objects # Synopsis From 894cbe43bc5caa339dab7f8af31bf065020c9f8d Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 31 Aug 2023 20:50:22 +0200 Subject: [PATCH 046/139] don't invent terms yet --- doc/manual/src/command-ref/nix-store/realise.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index 680e54ffa..b97c1f735 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -15,7 +15,7 @@ Each of *paths* is processed as follows: 1. If it is not [valid], substitute the store derivation file itself. 2. Realise its [output paths]: - Try to fetch from [substituters] the [store objects] associated with the output paths in the store derivation's [closure]. - - With [content-addressed derivations] (experimental): Determine the output paths to realise by querying build certificates in the [Nix database]. + - With [content-addressed derivations] (experimental): Determine the output paths to realise by querying content-addressed realisation entries in the [Nix database]. - For any store paths that cannot be substituted, produce the required store objects by first realising all outputs of the derivation's dependencies and then running the derivation's [`builder`](@docroot@/language/derivations.md#attr-builder) executable. - Otherwise, and if the path is not already valid: Try to fetch the associated [store objects] in the path's [closure] from [substituters]. From d38a539437943f4ae6b1f72321c7dc29559547c5 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 31 Aug 2023 20:50:50 +0200 Subject: [PATCH 047/139] make description open-ended, add TODO Co-authored-by: Robert Hensing --- doc/manual/src/command-ref/nix-store/realise.md | 2 +- doc/manual/src/glossary.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index b97c1f735..a421a4220 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -16,7 +16,7 @@ Each of *paths* is processed as follows: 2. Realise its [output paths]: - Try to fetch from [substituters] the [store objects] associated with the output paths in the store derivation's [closure]. - With [content-addressed derivations] (experimental): Determine the output paths to realise by querying content-addressed realisation entries in the [Nix database]. - - For any store paths that cannot be substituted, produce the required store objects by first realising all outputs of the derivation's dependencies and then running the derivation's [`builder`](@docroot@/language/derivations.md#attr-builder) executable. + - For any store paths that cannot be substituted, produce the required store objects. This involves first realising all outputs of the derivation's dependencies and then running the derivation's [`builder`](@docroot@/language/derivations.md#attr-builder) executable. - Otherwise, and if the path is not already valid: Try to fetch the associated [store objects] in the path's [closure] from [substituters]. If no substitutes are available and no store derivation is given, realisation fails. diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 8f0b25c89..57a3334dd 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -33,7 +33,7 @@ Ensure a [store path] is [valid][validity]. - This means either running the [`builder`](@docroot@/language/derivations.md#attr-builder) executable as specified in the corresponding [derivation], or fetching a pre-built [store object] from a [substituter], or delegating to a [remote builder](@docroot@/advanced-topics/distributed-builds.html) and retrieving the outputs. + This means either running the [`builder`](@docroot@/language/derivations.md#attr-builder) executable as specified in the corresponding [derivation], or fetching a pre-built [store object] from a [substituter], or delegating to a [remote builder](@docroot@/advanced-topics/distributed-builds.html) and retrieving the outputs. See [`nix-build`](./command-ref/nix-build.md) and [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md). From 1ac181759d975e4faeaf9083259287562009b4ec Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 31 Aug 2023 21:25:33 +0200 Subject: [PATCH 048/139] revert some random change --- doc/manual/src/command-ref/nix-store/opt-common.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/command-ref/nix-store/opt-common.md b/doc/manual/src/command-ref/nix-store/opt-common.md index 104b20076..dd9a6bf21 100644 --- a/doc/manual/src/command-ref/nix-store/opt-common.md +++ b/doc/manual/src/command-ref/nix-store/opt-common.md @@ -1,6 +1,6 @@ -# Command options +# Options -The following options are allowed for all operations in the `nix-store` command, but may not always have an effect. +The following options are allowed for all `nix-store` operations, but may not always have an effect. - [`--add-root`](#opt-add-root) *path* From 539cc5e5f00c0d524dec6e73b08ab8cb0f5a9630 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20=C4=8Cun=C3=A1t?= Date: Thu, 22 Jun 2023 11:08:56 +0200 Subject: [PATCH 049/139] flake: update nixpkgs: 22.11 -> 23.05 The lowdown input can't be updated; `nix build` would fail to find it. Co-authored-by: Robert Hensing --- flake.lock | 8 ++++---- flake.nix | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index 1d2aab5ed..2bc503258 100644 --- a/flake.lock +++ b/flake.lock @@ -34,16 +34,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1670461440, - "narHash": "sha256-jy1LB8HOMKGJEGXgzFRLDU1CBGL0/LlkolgnqIsF0D8=", + "lastModified": 1693494129, + "narHash": "sha256-YrHlSbniFmhcz0ORe8MMFttifKR4hTRzyX2OQUO9VxA=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "04a75b2eecc0acf6239acf9dd04485ff8d14f425", + "rev": "22a584b861ab31a2dca52121999079832f0e0f73", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-22.11-small", + "ref": "nixos-23.05-small", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index 36ba61484..49aa9c4ce 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,7 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-22.11-small"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; From 5c95b32c461da645826ff6bf248183f066c57b20 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 1 Sep 2023 14:49:49 +0200 Subject: [PATCH 050/139] Fix warning 'catching polymorphic type by value' --- src/libstore/build/create-derivation-and-realise-goal.cc | 2 +- src/libutil/util.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/build/create-derivation-and-realise-goal.cc b/src/libstore/build/create-derivation-and-realise-goal.cc index b01042f00..60f67956d 100644 --- a/src/libstore/build/create-derivation-and-realise-goal.cc +++ b/src/libstore/build/create-derivation-and-realise-goal.cc @@ -96,7 +96,7 @@ void CreateDerivationAndRealiseGoal::getDerivation() auto drvPath = StorePath::dummy; try { drvPath = resolveDerivedPath(worker.store, *drvReq); - } catch (MissingRealisation) { + } catch (MissingRealisation &) { return std::nullopt; } return worker.evalStore.isValidPath(drvPath) || worker.store.isValidPath(drvPath) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index f24a6e165..5a10c69e2 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -60,7 +60,7 @@ void initLibUtil() { bool caught = false; try { throwExceptionSelfCheck(); - } catch (nix::Error _e) { + } catch (const nix::Error & _e) { caught = true; } // This is not actually the main point of this check, but let's make sure anyway: From 73f6407eeaf0018519be5d4f9dcd87537450d277 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Sep 2023 22:52:37 +0000 Subject: [PATCH 051/139] Bump actions/checkout from 3 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/backport.yml | 2 +- .github/workflows/ci.yml | 8 ++++---- .github/workflows/hydra_status.yml | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 62bef6322..12c60c649 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -14,7 +14,7 @@ jobs: if: github.repository_owner == 'NixOS' && github.event.pull_request.merged == true && (github.event_name != 'labeled' || startsWith('backport', github.event.label.name)) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} # required to find all branches diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3a17d106..bcda8a77a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: runs-on: ${{ matrix.os }} timeout-minutes: 60 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - uses: cachix/install-nix-action@v22 @@ -58,7 +58,7 @@ jobs: outputs: installerURL: ${{ steps.prepare-installer.outputs.installerURL }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV @@ -82,7 +82,7 @@ jobs: os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - uses: cachix/install-nix-action@v22 with: @@ -108,7 +108,7 @@ jobs: needs.check_secrets.outputs.docker == 'true' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - uses: cachix/install-nix-action@v22 diff --git a/.github/workflows/hydra_status.yml b/.github/workflows/hydra_status.yml index 38a9c0877..2a7574747 100644 --- a/.github/workflows/hydra_status.yml +++ b/.github/workflows/hydra_status.yml @@ -13,7 +13,7 @@ jobs: if: github.repository_owner == 'NixOS' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - run: bash scripts/check-hydra-status.sh From 87508b1065c38d77870f7894a64f4edac162f3bb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Sep 2023 22:52:40 +0000 Subject: [PATCH 052/139] Bump cachix/install-nix-action from 22 to 23 Bumps [cachix/install-nix-action](https://github.com/cachix/install-nix-action) from 22 to 23. - [Release notes](https://github.com/cachix/install-nix-action/releases) - [Commits](https://github.com/cachix/install-nix-action/compare/v22...v23) --- updated-dependencies: - dependency-name: cachix/install-nix-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3a17d106..e024a851e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v22 + - uses: cachix/install-nix-action@v23 with: # The sandbox would otherwise be disabled by default on Darwin extra_nix_config: "sandbox = true" @@ -62,7 +62,7 @@ jobs: with: fetch-depth: 0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v22 + - uses: cachix/install-nix-action@v23 with: install_url: https://releases.nixos.org/nix/nix-2.13.3/install - uses: cachix/cachix-action@v12 @@ -84,7 +84,7 @@ jobs: steps: - uses: actions/checkout@v3 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v22 + - uses: cachix/install-nix-action@v23 with: install_url: '${{needs.installer.outputs.installerURL}}' install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" @@ -111,7 +111,7 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v22 + - uses: cachix/install-nix-action@v23 with: install_url: https://releases.nixos.org/nix/nix-2.13.3/install - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV From cc388fbc3a084a1e400c7c65df08b8dc6281056d Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 30 May 2023 23:00:16 +0200 Subject: [PATCH 053/139] remove maintainers checklist in PR template maintainers are not really using it, and it produces a lot of noise when opening PRs. --- .github/PULL_REQUEST_TEMPLATE.md | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 4488c7b7d..217b19108 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -10,24 +10,6 @@ -# Checklist for maintainers - - - -Maintainers: tick if completed or explain if not relevant - - - [ ] agreed on idea - - [ ] agreed on implementation strategy - - [ ] tests, as appropriate - - functional tests - `tests/**.sh` - - unit tests - `src/*/tests` - - integration tests - `tests/nixos/*` - - [ ] documentation in the manual - - [ ] documentation in the internal API docs - - [ ] code and comments are self-explanatory - - [ ] commit message explains why the change was made - - [ ] new feature or incompatible change: updated release notes - # Priorities Add :+1: to [pull requests you find important](https://github.com/NixOS/nix/pulls?q=is%3Aopen+sort%3Areactions-%2B1-desc). From 4f2b949ba808e2161f652cd914cba2517f1faea5 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 13 Jul 2023 21:57:32 +0200 Subject: [PATCH 054/139] reorder list items --- CONTRIBUTING.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4a72a8eac..6c9d97c04 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,8 +30,8 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). 2. Search for related issues that cover what you're going to work on. It could help to mention there that you will work on the issue. - Issues labeled ["good first issue"](https://github.com/NixOS/nix/labels/good-first-issue) should be relatively easy to fix and are likely to get merged quickly. - Pull requests addressing issues labeled ["idea approved"](https://github.com/NixOS/nix/labels/idea%20approved) are especially welcomed by maintainers and will receive prioritised review. + Issues labeled [good first issue](https://github.com/NixOS/nix/labels/good-first-issue) should be relatively easy to fix and are likely to get merged quickly. + Pull requests addressing issues labeled [idea approved](https://github.com/NixOS/nix/labels/idea%20approved) are especially welcomed by maintainers and will receive prioritised review. 3. Check the [Nix reference manual](https://nixos.org/manual/nix/unstable/contributing/hacking.html) for information on building Nix and running its tests. @@ -40,10 +40,10 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). 4. Make your changes! 5. [Create a pull request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request) for your changes. - * [Mark the pull request as draft](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/changing-the-stage-of-a-pull-request) if you're not done with the changes. - * Make sure to have [a clean history of commits on your branch by using rebase](https://www.digitalocean.com/community/tutorials/how-to-rebase-and-update-a-pull-request). * Link related issues in your pull request to inform interested parties and future contributors about your change. + * Make sure to have [a clean history of commits on your branch by using rebase](https://www.digitalocean.com/community/tutorials/how-to-rebase-and-update-a-pull-request). If your pull request closes one or multiple issues, note that in the description using `Closes: #`, as it will then happen automatically when your change is merged. + * [Mark the pull request as draft](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/changing-the-stage-of-a-pull-request) if you're not done with the changes. 6. Do not expect your pull request to be reviewed immediately. Nix maintainers follow a [structured process for reviews and design decisions](https://github.com/NixOS/nix/tree/master/maintainers#project-board-protocol), which may or may not prioritise your work. From 3a9c1dc8a3b872adaf30562fb28697b58be4be2b Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 13 Jul 2023 21:57:41 +0200 Subject: [PATCH 055/139] add checklist to contribution guide --- CONTRIBUTING.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6c9d97c04..facbf0eb0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,6 +48,19 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). 6. Do not expect your pull request to be reviewed immediately. Nix maintainers follow a [structured process for reviews and design decisions](https://github.com/NixOS/nix/tree/master/maintainers#project-board-protocol), which may or may not prioritise your work. + Following this checklist will make the process smoother for everyone: + + - [ ] Fixes an [idea approved](https://github.com/NixOS/nix/labels/idea%20approved) issue + - [ ] Tests, as appropriate: + - Functional tests – [`tests/**.sh`](./tests) + - Unit tests – [`src/*/tests`](./src/) + - Integration tests – [`tests/nixos/*`](./tests/nixos) + - [ ] User documentation in the [manual](..doc/manual/src) + - [ ] API documentation in header files + - [ ] Code and comments are self-explanatory + - [ ] Commit message explains **why** the change was made + - [ ] New feature or incompatible change: updated [release notes](./doc/manual/src/release-notes/rl-next.md) + 7. If you need additional feedback or help to getting pull request into shape, ask other contributors using [@mentions](https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax#mentioning-people-and-teams). ## Making changes to the Nix manual From 7ff43435f9b23c9e0d445427cc37b147c84b056f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 4 Sep 2023 18:15:32 -0400 Subject: [PATCH 056/139] Unit test some worker protocol serializers Continue with the characterization testing idioms begun in c70484454f52a3bba994ac0c155b4a6f35a5013c, but this time for unit tests. Co-authored-by: Andreas Rammhold --- Makefile | 1 + doc/manual/src/contributing/testing.md | 73 ++++++++- flake.nix | 1 + mk/programs.mk | 2 +- src/libstore/tests/worker-protocol.cc | 139 ++++++++++++++++++ src/libstore/worker-protocol-impl.hh | 16 ++ src/libstore/worker-protocol.hh | 4 + .../worker-protocol/content-address.bin | Bin 0 -> 208 bytes .../libstore/worker-protocol/derived-path.bin | Bin 0 -> 120 bytes .../libstore/worker-protocol/store-path.bin | Bin 0 -> 120 bytes .../libstore/worker-protocol/string.bin | Bin 0 -> 88 bytes 11 files changed, 228 insertions(+), 8 deletions(-) create mode 100644 src/libstore/tests/worker-protocol.cc create mode 100644 unit-test-data/libstore/worker-protocol/content-address.bin create mode 100644 unit-test-data/libstore/worker-protocol/derived-path.bin create mode 100644 unit-test-data/libstore/worker-protocol/store-path.bin create mode 100644 unit-test-data/libstore/worker-protocol/string.bin diff --git a/Makefile b/Makefile index 31b54b93d..2eabeb077 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,7 @@ makefiles = \ -include Makefile.config ifeq ($(tests), yes) +UNIT_TEST_ENV = _NIX_TEST_UNIT_DATA=unit-test-data makefiles += \ src/libutil/tests/local.mk \ src/libstore/tests/local.mk \ diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index cd94d5cfb..329b34575 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -2,14 +2,70 @@ ## Unit-tests -The unit-tests for each Nix library (`libexpr`, `libstore`, etc..) are defined -under `src/{library_name}/tests` using the -[googletest](https://google.github.io/googletest/) and -[rapidcheck](https://github.com/emil-e/rapidcheck) frameworks. +The unit tests are defined using the [googletest] and [rapidcheck] frameworks. + +[googletest]: https://google.github.io/googletest/ +[rapidcheck]: https://github.com/emil-e/rapidcheck + +### Source and header layout + +> An example of some files, demonstrating much of what is described below +> +> ``` +> src +> ├── libexpr +> │ ├── value/context.hh +> │ ├── value/context.cc +> │ │ +> │ … +> └── tests +> │ ├── value/context.hh +> │ ├── value/context.cc +> │ │ +> │ … +> │ +> ├── unit-test-data +> │ ├── libstore +> │ │ ├── worker-protocol/content-address.bin +> │ │ … +> │ … +> … +> ``` + +The unit tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `src/${library_shortname}/tests` within the directory for the library (`src/${library_shortname}`). + +The data is in `unit-test-data`, with one subdir per library, with the same name as where the code goes. +For example, `libnixstore` code is in `src/libstore`, and its test data is in `unit-test-data/libstore`. +The path to the `unit-test-data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. + +> **Note** +> Due to the way googletest works, downstream unit test executables will actually include and re-run upstream library tests. +> Therefore it is important that the same value for `_NIX_TEST_UNIT_DATA` be used with the tests for each library. +> That is why we have the test data nested within a single `unit-test-data` directory. + +### Running tests You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option, or the `GTEST_FILTER` environment variable. +### Characterization testing + +See [below](#characterization-testing-1) for a broader discussion of characterization testing. + +Like with the functional characterization, `_NIX_TEST_ACCEPT=1` is also used. +For example: +```shell-session +$ _NIX_TEST_ACCEPT=1 make libstore-tests-exe_RUN +... +[ SKIPPED ] WorkerProtoTest.string_read +[ SKIPPED ] WorkerProtoTest.string_write +[ SKIPPED ] WorkerProtoTest.storePath_read +[ SKIPPED ] WorkerProtoTest.storePath_write +... +``` +will regenerate the "golden master" expected result for the `libnixstore` characterization tests. +The characterization tests will mark themselves "skipped" since they regenerated the expected result instead of actually testing anything. + ## Functional tests The functional tests reside under the `tests` directory and are listed in `tests/local.mk`. @@ -124,9 +180,12 @@ This technique is to include the exact output/behavior of a former version of Ni For example, this technique is used for the language tests, to check both the printed final value if evaluation was successful, and any errors and warnings encountered. It is frequently useful to regenerate the expected output. -To do that, rerun the failed test with `_NIX_TEST_ACCEPT=1`. -(At least, this is the convention we've used for `tests/lang.sh`. -If we add more characterization testing we should always strive to be consistent.) +To do that, rerun the failed test(s) with `_NIX_TEST_ACCEPT=1`. +For example: +```bash +_NIX_TEST_ACCEPT=1 make tests/lang.sh.test +``` +This convention is shared with the [characterization unit tests](#characterization-testing-1) too. An interesting situation to document is the case when these tests are "overfitted". The language tests are, again, an example of this. diff --git a/flake.nix b/flake.nix index cf7c1fa12..ec6ab48bd 100644 --- a/flake.nix +++ b/flake.nix @@ -75,6 +75,7 @@ ./precompiled-headers.h ./src ./tests + ./unit-test-data ./COPYING ./scripts/local.mk (fileset.fileFilter (f: lib.strings.hasPrefix "nix-profile" f.name) ./scripts) diff --git a/mk/programs.mk b/mk/programs.mk index 1ee1d3fa5..a88d9d949 100644 --- a/mk/programs.mk +++ b/mk/programs.mk @@ -87,6 +87,6 @@ define build-program # Phony target to run this program (typically as a dependency of 'check'). .PHONY: $(1)_RUN $(1)_RUN: $$($(1)_PATH) - $(trace-test) $$($(1)_PATH) + $(trace-test) $$(UNIT_TEST_ENV) $$($(1)_PATH) endef diff --git a/src/libstore/tests/worker-protocol.cc b/src/libstore/tests/worker-protocol.cc new file mode 100644 index 000000000..4a6ccf7c0 --- /dev/null +++ b/src/libstore/tests/worker-protocol.cc @@ -0,0 +1,139 @@ +#include + +#include +#include + +#include "worker-protocol.hh" +#include "worker-protocol-impl.hh" +#include "derived-path.hh" +#include "tests/libstore.hh" + +namespace nix { + +class WorkerProtoTest : public LibStoreTest +{ +public: + Path unitTestData = getEnv("_NIX_TEST_UNIT_DATA").value() + "/libstore/worker-protocol"; + + bool testAccept() { + return getEnv("_NIX_TEST_ACCEPT") == "1"; + } + + Path goldenMaster(std::string_view testStem) { + return unitTestData + "/" + testStem + ".bin"; + } + + /** + * Golden test for `T` reading + */ + template + void readTest(PathView testStem, T value) + { + if (testAccept()) + { + GTEST_SKIP() << "Cannot read golden master because another test is also updating it"; + } + else + { + auto expected = readFile(goldenMaster(testStem)); + + T got = ({ + StringSource from { expected }; + WorkerProto::Serialise::read( + *store, + WorkerProto::ReadConn { .from = from }); + }); + + ASSERT_EQ(got, value); + } + } + + /** + * Golden test for `T` write + */ + template + void writeTest(PathView testStem, const T & value) + { + auto file = goldenMaster(testStem); + + StringSink to; + WorkerProto::write( + *store, + WorkerProto::WriteConn { .to = to }, + value); + + if (testAccept()) + { + createDirs(dirOf(file)); + writeFile(file, to.s); + GTEST_SKIP() << "Updating golden master"; + } + else + { + auto expected = readFile(file); + ASSERT_EQ(to.s, expected); + } + } +}; + +#define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \ + TEST_F(WorkerProtoTest, NAME ## _read) { \ + readTest(STEM, VALUE); \ + } \ + TEST_F(WorkerProtoTest, NAME ## _write) { \ + writeTest(STEM, VALUE); \ + } + +CHARACTERIZATION_TEST( + string, + "string", + (std::tuple { + "", + "hi", + "white rabbit", + "大白兔", + "oh no \0\0\0 what was that!", + })) + +CHARACTERIZATION_TEST( + storePath, + "store-path", + (std::tuple { + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo-bar" }, + })) + +CHARACTERIZATION_TEST( + contentAddress, + "content-address", + (std::tuple { + ContentAddress { + .method = TextIngestionMethod {}, + .hash = hashString(HashType::htSHA256, "Derive(...)"), + }, + ContentAddress { + .method = FileIngestionMethod::Flat, + .hash = hashString(HashType::htSHA1, "blob blob..."), + }, + ContentAddress { + .method = FileIngestionMethod::Recursive, + .hash = hashString(HashType::htSHA256, "(...)"), + }, + })) + +CHARACTERIZATION_TEST( + derivedPath, + "derived-path", + (std::tuple { + DerivedPath::Opaque { + .path = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" }, + }, + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }), + .outputs = OutputsSpec::Names { "x", "y" }, + }, + })) + +} diff --git a/src/libstore/worker-protocol-impl.hh b/src/libstore/worker-protocol-impl.hh index d3d2792ff..4f797f95a 100644 --- a/src/libstore/worker-protocol-impl.hh +++ b/src/libstore/worker-protocol-impl.hh @@ -75,4 +75,20 @@ void WorkerProto::Serialise>::write(const Store & store, WorkerPr } } +template +std::tuple WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) +{ + return std::tuple { + WorkerProto::Serialise::read(store, conn)..., + }; +} + +template +void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::tuple & res) +{ + std::apply([&](const Us &... args) { + (WorkerProto::Serialise::write(store, conn, args), ...); + }, res); +} + } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index ff762c924..70a5bddb9 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -28,6 +28,8 @@ class Store; struct Source; // items being serialised +class StorePath; +struct ContentAddress; struct DerivedPath; struct DrvOutput; struct Realisation; @@ -220,6 +222,8 @@ template MAKE_WORKER_PROTO(std::vector); template MAKE_WORKER_PROTO(std::set); +template +MAKE_WORKER_PROTO(std::tuple); template #define X_ std::map diff --git a/unit-test-data/libstore/worker-protocol/content-address.bin b/unit-test-data/libstore/worker-protocol/content-address.bin new file mode 100644 index 0000000000000000000000000000000000000000..8f14bcdb3e50cc72d87106ce159a6fd7b9f75713 GIT binary patch literal 208 zcmY+;K@P$o5QSmy;|7WYrYjRqQfMm$+LWPzDW_MfG4bucKks(>Y#V56lkFOiEzi24 zUMdC5VUCa86OJ&gL0BF^B{HBQEY%f|I<6v7#q+l_PBirI5N~-md%37WE|g33QTE2k zc~-WF&R}7Oxc@o)T?ojpD!+*5=xpO;%IoCKSV5FMh={>xePK|8<${>E)*huNHZ(pX literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/worker-protocol/derived-path.bin b/unit-test-data/libstore/worker-protocol/derived-path.bin new file mode 100644 index 0000000000000000000000000000000000000000..bb1a81ac6d096cddf510a457e7506e6a81ee66a3 GIT binary patch literal 120 zcmdOAfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7Eu*IT`wr5{vXwipmr#bSfDDBbg*z literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/worker-protocol/store-path.bin b/unit-test-data/libstore/worker-protocol/store-path.bin new file mode 100644 index 0000000000000000000000000000000000000000..3fc05f2981d3398ab30ec34125c903cdcefa8729 GIT binary patch literal 120 tcmdOAfB^lx%nJSDlKi4n{dB`}^NdR4LR_?NT7Eu*F?srQlM;)-IsqP{BM|@q literal 0 HcmV?d00001 diff --git a/unit-test-data/libstore/worker-protocol/string.bin b/unit-test-data/libstore/worker-protocol/string.bin new file mode 100644 index 0000000000000000000000000000000000000000..aa7b5a604745b735473ec34283177e69320776e4 GIT binary patch literal 88 zcmZQzfB+^aoskJ)@Id+H8JQ)i3Pp)YNtq=eAx^0H( Date: Tue, 5 Sep 2023 11:16:39 -0400 Subject: [PATCH 057/139] Test and begin documentation of the ATerm format for derivations Wanted to do this before the last dynamic derivations PR when I introduce a variation, to make sure I wasn't changing the old version by mistake. --- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/protocols/derivation-aterm.md | 9 ++ src/libstore/tests/derivation.cc | 119 ++++++++++++------- 3 files changed, 83 insertions(+), 46 deletions(-) create mode 100644 doc/manual/src/protocols/derivation-aterm.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 6c599abcf..3de053a17 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -100,6 +100,7 @@ - [File System Object](architecture/file-system-object.md) - [Protocols](protocols/protocols.md) - [Serving Tarball Flakes](protocols/tarball-fetcher.md) + - [Derivation on-disk "ATerm" format](protocols/derivation-aterm.md) - [Glossary](glossary.md) - [Contributing](contributing/contributing.md) - [Hacking](contributing/hacking.md) diff --git a/doc/manual/src/protocols/derivation-aterm.md b/doc/manual/src/protocols/derivation-aterm.md new file mode 100644 index 000000000..2377e51ae --- /dev/null +++ b/doc/manual/src/protocols/derivation-aterm.md @@ -0,0 +1,9 @@ +# Derivation on-disk "ATerm" format + +For historical reasons, [derivations](@docroot@/glossary.md#gloss-store-derivation) are stored on-disk in [ATerm](https://homepages.cwi.nl/~daybuild/daily-books/technology/aterm-guide/aterm-guide.html) format. + +Derivations are serialised in the following format: + +``` +Derive(...) +``` diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc index 0e28c1f08..7fd7f9278 100644 --- a/src/libstore/tests/derivation.cc +++ b/src/libstore/tests/derivation.cc @@ -143,26 +143,76 @@ TEST_JSON(ImpureDerivationTest, impure, #undef TEST_JSON -#define TEST_JSON(NAME, STR, VAL, DRV_NAME) \ - TEST_F(DerivationTest, Derivation_ ## NAME ## _to_json) { \ - using nlohmann::literals::operator "" _json; \ - ASSERT_EQ( \ - STR ## _json, \ - (Derivation { VAL }).toJSON(*store)); \ - } \ - \ - TEST_F(DerivationTest, Derivation_ ## NAME ## _from_json) { \ - using nlohmann::literals::operator "" _json; \ - ASSERT_EQ( \ - Derivation { VAL }, \ - Derivation::fromJSON( \ - *store, \ - STR ## _json)); \ +#define TEST_JSON(NAME, STR, VAL) \ + TEST_F(DerivationTest, Derivation_ ## NAME ## _to_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + STR ## _json, \ + (VAL).toJSON(*store)); \ + } \ + \ + TEST_F(DerivationTest, Derivation_ ## NAME ## _from_json) { \ + using nlohmann::literals::operator "" _json; \ + ASSERT_EQ( \ + (VAL), \ + Derivation::fromJSON( \ + *store, \ + STR ## _json)); \ } +#define TEST_ATERM(NAME, STR, VAL, DRV_NAME) \ + TEST_F(DerivationTest, Derivation_ ## NAME ## _to_aterm) { \ + ASSERT_EQ( \ + STR, \ + (VAL).unparse(*store, false)); \ + } \ + \ + TEST_F(DerivationTest, Derivation_ ## NAME ## _from_aterm) { \ + auto parsed = parseDerivation( \ + *store, \ + STR, \ + DRV_NAME); \ + ASSERT_EQ( \ + (VAL).toJSON(*store), \ + parsed.toJSON(*store)); \ + ASSERT_EQ( \ + (VAL), \ + parsed); \ + } + +Derivation makeSimpleDrv(const Store & store) { + Derivation drv; + drv.name = "simple-derivation"; + drv.inputSrcs = { + store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), + }; + drv.inputDrvs = { + { + store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), + { + "cat", + "dog", + }, + }, + }; + drv.platform = "wasm-sel4"; + drv.builder = "foo"; + drv.args = { + "bar", + "baz", + }; + drv.env = { + { + "BIG_BAD", + "WOLF", + }, + }; + return drv; +} + TEST_JSON(simple, R"({ - "name": "my-derivation", + "name": "simple-derivation", "inputSrcs": [ "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" ], @@ -183,37 +233,14 @@ TEST_JSON(simple, }, "outputs": {} })", - ({ - Derivation drv; - drv.name = "my-derivation"; - drv.inputSrcs = { - store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), - }; - drv.inputDrvs = { - { - store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), - { - "cat", - "dog", - }, - } - }; - drv.platform = "wasm-sel4"; - drv.builder = "foo"; - drv.args = { - "bar", - "baz", - }; - drv.env = { - { - "BIG_BAD", - "WOLF", - }, - }; - drv; - }), - "drv-name") + makeSimpleDrv(*store)) + +TEST_ATERM(simple, + R"(Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))", + makeSimpleDrv(*store), + "simple-derivation") #undef TEST_JSON +#undef TEST_ATERM } From 2b3a17820f202ea2108184a399f3c573ca11c1b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christina=20S=C3=B8rensen?= Date: Wed, 6 Sep 2023 04:19:40 +0000 Subject: [PATCH 058/139] Fix globals.hh typo --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 7009f6bb8..cf10edebd 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -554,7 +554,7 @@ public: R"( This option determines the maximum size of the `tmpfs` filesystem mounted on `/dev/shm` in Linux sandboxes. For the format, see the - description of the `size` option of `tmpfs` in mount8. The default + description of the `size` option of `tmpfs` in mount(8). The default is `50%`. )"}; From 5c23d3a90cb1a76a0ea464be380bb16dc9f999c7 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 6 Sep 2023 09:09:02 +0200 Subject: [PATCH 059/139] disambiguate output from output path --- doc/manual/src/glossary.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 57a3334dd..b7c340d36 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -197,9 +197,15 @@ [closure]: #gloss-closure +- [output]{#gloss-output} + + A [store object] produced by a [derivation]. + + [output]: #gloss-output + - [output path]{#gloss-output-path} - A [store path] produced by a [derivation]. + The [store path] to the [output] of a [derivation]. [output path]: #gloss-output-path From 02c2679f0e94013b5cae6305e0b15fce8d971eb8 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 6 Sep 2023 09:49:00 +0200 Subject: [PATCH 060/139] dedent common options listing; one sentence per line this is a pure reformatting, contents were not changed one sentence per line makes reviewing diffs and making suggestions much more convenient. the indentation was an artifat of the DocBook migration. --- doc/manual/src/command-ref/opt-common.md | 335 +++++++++++------------ 1 file changed, 161 insertions(+), 174 deletions(-) diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index 54c0a1d0d..0647228c2 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -2,217 +2,204 @@ Most Nix commands accept the following command-line options: - - [`--help`](#opt-help)\ - Prints out a summary of the command syntax and exits. +- [`--help`](#opt-help) - - [`--version`](#opt-version)\ - Prints out the Nix version number on standard output and exits. + Prints out a summary of the command syntax and exits. - - [`--verbose`](#opt-verbose) / `-v`\ - Increases the level of verbosity of diagnostic messages printed on - standard error. For each Nix operation, the information printed on - standard output is well-defined; any diagnostic information is - printed on standard error, never on standard output. +- [`--version`](#opt-version) - This option may be specified repeatedly. Currently, the following - verbosity levels exist: + Prints out the Nix version number on standard output and exits. - - 0\ - “Errors only”: only print messages explaining why the Nix - invocation failed. +- [`--verbose`](#opt-verbose) / `-v` - - 1\ - “Informational”: print *useful* messages about what Nix is - doing. This is the default. + Increases the level of verbosity of diagnostic messages printed on standard error. + For each Nix operation, the information printed on standard output is well-defined; + any diagnostic information is printed on standard error, never on standard output. - - 2\ - “Talkative”: print more informational messages. + This option may be specified repeatedly. + Currently, the following verbosity levels exist: - - 3\ - “Chatty”: print even more informational messages. + - `0` “Errors only” - - 4\ - “Debug”: print debug information. + Only print messages explaining why the Nix invocation failed. - - 5\ - “Vomit”: print vast amounts of debug information. + - `1` “Informational” - - [`--quiet`](#opt-quiet)\ - Decreases the level of verbosity of diagnostic messages printed on - standard error. This is the inverse option to `-v` / `--verbose`. + Print *useful* messages about what Nix is doing. + This is the default. - This option may be specified repeatedly. See the previous verbosity - levels list. + - `2` “Talkative” - - [`--log-format`](#opt-log-format) *format*\ - This option can be used to change the output of the log format, with - *format* being one of: + Print more informational messages. - - raw\ - This is the raw format, as outputted by nix-build. + - `3` “Chatty” - - internal-json\ - Outputs the logs in a structured manner. + Print even more informational messages. - > **Warning** - > - > While the schema itself is relatively stable, the format of - > the error-messages (namely of the `msg`-field) can change - > between releases. + - `4` “Debug” + + Print debug information. - - bar\ - Only display a progress bar during the builds. + - `5` “Vomit” - - bar-with-logs\ - Display the raw logs, with the progress bar at the bottom. + Print vast amounts of debug information. - - [`--no-build-output`](#opt-no-build-output) / `-Q`\ - By default, output written by builders to standard output and - standard error is echoed to the Nix command's standard error. This - option suppresses this behaviour. Note that the builder's standard - output and error are always written to a log file in - `prefix/nix/var/log/nix`. +- [`--quiet`](#opt-quiet) - - [`--max-jobs`](#opt-max-jobs) / `-j` *number*\ - Sets the maximum number of build jobs that Nix will perform in - parallel to the specified number. Specify `auto` to use the number - of CPUs in the system. The default is specified by the `max-jobs` - configuration setting, which itself defaults to `1`. A higher - value is useful on SMP systems or to exploit I/O latency. + Decreases the level of verbosity of diagnostic messages printed on standard error. + This is the inverse option to `-v` / `--verbose`. - Setting it to `0` disallows building on the local machine, which is - useful when you want builds to happen only on remote builders. + This option may be specified repeatedly. + See the previous verbosity levels list. - - [`--cores`](#opt-cores)\ - Sets the value of the `NIX_BUILD_CORES` environment variable in - the invocation of builders. Builders can use this variable at - their discretion to control the maximum amount of parallelism. For - instance, in Nixpkgs, if the derivation attribute - `enableParallelBuilding` is set to `true`, the builder passes the - `-jN` flag to GNU Make. It defaults to the value of the `cores` - configuration setting, if set, or `1` otherwise. The value `0` - means that the builder should use all available CPU cores in the - system. +- [`--log-format`](#opt-log-format) *format* - - [`--max-silent-time`](#opt-max-silent-time)\ - Sets the maximum number of seconds that a builder can go without - producing any data on standard output or standard error. The - default is specified by the `max-silent-time` configuration - setting. `0` means no time-out. + This option can be used to change the output of the log format, with *format* being one of: - - [`--timeout`](#opt-timeout)\ - Sets the maximum number of seconds that a builder can run. The - default is specified by the `timeout` configuration setting. `0` - means no timeout. + - `raw` - - [`--keep-going`](#opt-keep-going) / `-k`\ - Keep going in case of failed builds, to the greatest extent - possible. That is, if building an input of some derivation fails, - Nix will still build the other inputs, but not the derivation - itself. Without this option, Nix stops if any build fails (except - for builds of substitutes), possibly killing builds in progress (in - case of parallel or distributed builds). + This is the raw format, as outputted by nix-build. - - [`--keep-failed`](#opt-keep-failed) / `-K`\ - Specifies that in case of a build failure, the temporary directory - (usually in `/tmp`) in which the build takes place should not be - deleted. The path of the build directory is printed as an - informational message. + - `internal-json` - - [`--fallback`](#opt-fallback)\ - Whenever Nix attempts to build a derivation for which substitutes - are known for each output path, but realising the output paths - through the substitutes fails, fall back on building the derivation. + Outputs the logs in a structured manner. - The most common scenario in which this is useful is when we have - registered substitutes in order to perform binary distribution from, - say, a network repository. If the repository is down, the - realisation of the derivation will fail. When this option is - specified, Nix will build the derivation instead. Thus, installation - from binaries falls back on installation from source. This option is - not the default since it is generally not desirable for a transient - failure in obtaining the substitutes to lead to a full build from - source (with the related consumption of resources). + > **Warning** + > + > While the schema itself is relatively stable, the format of + > the error-messages (namely of the `msg`-field) can change + > between releases. - - [`--readonly-mode`](#opt-readonly-mode)\ - When this option is used, no attempt is made to open the Nix - database. Most Nix operations do need database access, so those - operations will fail. + - `bar` - - [`--arg`](#opt-arg) *name* *value*\ - This option is accepted by `nix-env`, `nix-instantiate`, - `nix-shell` and `nix-build`. When evaluating Nix expressions, the - expression evaluator will automatically try to call functions that - it encounters. It can automatically call functions for which every - argument has a [default - value](@docroot@/language/constructs.md#functions) (e.g., - `{ argName ? defaultValue }: ...`). With `--arg`, you can also - call functions that have arguments without a default value (or - override a default value). That is, if the evaluator encounters a - function with an argument named *name*, it will call it with value - *value*. + Only display a progress bar during the builds. - For instance, the top-level `default.nix` in Nixpkgs is actually a - function: + - `bar-with-logs` - ```nix - { # The system (e.g., `i686-linux') for which to build the packages. - system ? builtins.currentSystem - ... - }: ... - ``` + Display the raw logs, with the progress bar at the bottom. - So if you call this Nix expression (e.g., when you do `nix-env --install --attr - pkgname`), the function will be called automatically using the - value [`builtins.currentSystem`](@docroot@/language/builtins.md) for - the `system` argument. You can override this using `--arg`, e.g., - `nix-env --install --attr pkgname --arg system \"i686-freebsd\"`. (Note that - since the argument is a Nix string literal, you have to escape the - quotes.) +- [`--no-build-output`](#opt-no-build-output) / `-Q` - - [`--argstr`](#opt-argstr) *name* *value*\ - This option is like `--arg`, only the value is not a Nix - expression but a string. So instead of `--arg system - \"i686-linux\"` (the outer quotes are to keep the shell happy) you - can say `--argstr system i686-linux`. + By default, output written by builders to standard output and standard error is echoed to the Nix command's standard error. + This option suppresses this behaviour. + Note that the builder's standard output and error are always written to a log file in `prefix/nix/var/log/nix`. - - [`--attr`](#opt-attr) / `-A` *attrPath*\ - Select an attribute from the top-level Nix expression being - evaluated. (`nix-env`, `nix-instantiate`, `nix-build` and - `nix-shell` only.) The *attribute path* *attrPath* is a sequence - of attribute names separated by dots. For instance, given a - top-level Nix expression *e*, the attribute path `xorg.xorgserver` - would cause the expression `e.xorg.xorgserver` to be used. See - [`nix-env --install`](@docroot@/command-ref/nix-env/install.md) for some - concrete examples. +- [`--max-jobs`](#opt-max-jobs) / `-j` *number* - In addition to attribute names, you can also specify array indices. - For instance, the attribute path `foo.3.bar` selects the `bar` - attribute of the fourth element of the array in the `foo` attribute - of the top-level expression. + Sets the maximum number of build jobs that Nix will perform in parallel to the specified number. + Specify `auto` to use the number of CPUs in the system. + The default is specified by the `max-jobs` configuration setting, which itself defaults to `1`. + A higher value is useful on SMP systems or to exploit I/O latency. - - [`--expr`](#opt-expr) / `-E`\ - Interpret the command line arguments as a list of Nix expressions to - be parsed and evaluated, rather than as a list of file names of Nix - expressions. (`nix-instantiate`, `nix-build` and `nix-shell` only.) + Setting it to `0` disallows building on the local machine, which is useful when you want builds to happen only on remote builders. - For `nix-shell`, this option is commonly used to give you a shell in - which you can build the packages returned by the expression. If you - want to get a shell which contain the *built* packages ready for - use, give your expression to the `nix-shell --packages ` convenience flag - instead. +- [`--cores`](#opt-cores) - - [`-I`](#opt-I) *path*\ - Add an entry to the [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path). - This option may be given multiple times. - Paths added through `-I` take precedence over [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH). + Sets the value of the `NIX_BUILD_CORES` environment variable in the invocation of builders. + Builders can use this variable at their discretion to control the maximum amount of parallelism. + For instance, in Nixpkgs, if the derivation attribute `enableParallelBuilding` is set to `true`, the builder passes the `-jN` flag to GNU Make. + It defaults to the value of the `cores` configuration setting, if set, or `1` otherwise. + The value `0` means that the builder should use all available CPU cores in the system. - - [`--option`](#opt-option) *name* *value*\ - Set the Nix configuration option *name* to *value*. This overrides - settings in the Nix configuration file (see nix.conf5). +- [`--max-silent-time`](#opt-max-silent-time) - - [`--repair`](#opt-repair)\ - Fix corrupted or missing store paths by redownloading or rebuilding - them. Note that this is slow because it requires computing a - cryptographic hash of the contents of every path in the closure of - the build. Also note the warning under `nix-store --repair-path`. + Sets the maximum number of seconds that a builder can go without producing any data on standard output or standard error. + The default is specified by the `max-silent-time` configuration setting. + `0` means no time-out. + +- [`--timeout`](#opt-timeout) + + Sets the maximum number of seconds that a builder can run. + The default is specified by the `timeout` configuration setting. + `0` means no timeout. + +- [`--keep-going`](#opt-keep-going) / `-k` + + Keep going in case of failed builds, to the greatest extent possible. + That is, if building an input of some derivation fails, Nix will still build the other inputs, but not the derivation itself. + Without this option, Nix stops if any build fails (except for builds of substitutes), possibly killing builds in progress (in case of parallel or distributed builds). + +- [`--keep-failed`](#opt-keep-failed) / `-K` + + Specifies that in case of a build failure, the temporary directory (usually in `/tmp`) in which the build takes place should not be deleted. + The path of the build directory is printed as an informational message. + +- [`--fallback`](#opt-fallback) + + Whenever Nix attempts to build a derivation for which substitutes are known for each output path, but realising the output paths through the substitutes fails, fall back on building the derivation. + + The most common scenario in which this is useful is when we have registered substitutes in order to perform binary distribution from, say, a network repository. + If the repository is down, the realisation of the derivation will fail. + When this option is specified, Nix will build the derivation instead. + Thus, installation from binaries falls back on installation from source. + This option is not the default since it is generally not desirable for a transient failure in obtaining the substitutes to lead to a full build from source (with the related consumption of resources). + +- [`--readonly-mode`](#opt-readonly-mode) + + When this option is used, no attempt is made to open the Nix database. + Most Nix operations do need database access, so those operations will fail. + +- [`--arg`](#opt-arg) *name* *value* + + This option is accepted by `nix-env`, `nix-instantiate`, `nix-shell` and `nix-build`. + When evaluating Nix expressions, the expression evaluator will automatically try to call functions that it encounters. + It can automatically call functions for which every argument has a [default value](@docroot@/language/constructs.md#functions) (e.g., `{ argName ? defaultValue }: ...`). + + With `--arg`, you can also call functions that have arguments without a default value (or override a default value). + That is, if the evaluator encounters a function with an argument named *name*, it will call it with value *value*. + + For instance, the top-level `default.nix` in Nixpkgs is actually a function: + + ```nix + { # The system (e.g., `i686-linux') for which to build the packages. + system ? builtins.currentSystem + ... + }: ... + ``` + + So if you call this Nix expression (e.g., when you do `nix-env --install --attr pkgname`), the function will be called automatically using the value [`builtins.currentSystem`](@docroot@/language/builtins.md) for the `system` argument. + You can override this using `--arg`, e.g., `nix-env --install --attr pkgname --arg system \"i686-freebsd\"`. + (Note that since the argument is a Nix string literal, you have to escape the quotes.) + +- [`--argstr`](#opt-argstr) *name* *value* + + This option is like `--arg`, only the value is not a Nix expression but a string. + So instead of `--arg system \"i686-linux\"` (the outer quotes are to keep the shell happy) you can say `--argstr system i686-linux`. + +- [`--attr`](#opt-attr) / `-A` *attrPath* + + Select an attribute from the top-level Nix expression being evaluated. + (`nix-env`, `nix-instantiate`, `nix-build` and `nix-shell` only.) + The *attribute path* *attrPath* is a sequence of attribute names separated by dots. + For instance, given a top-level Nix expression *e*, the attribute path `xorg.xorgserver` would cause the expression `e.xorg.xorgserver` to be used. + See [`nix-env --install`](@docroot@/command-ref/nix-env/install.md) for some concrete examples. + + In addition to attribute names, you can also specify array indices. + For instance, the attribute path `foo.3.bar` selects the `bar` + attribute of the fourth element of the array in the `foo` attribute + of the top-level expression. + +- [`--expr`](#opt-expr) / `-E` + + Interpret the command line arguments as a list of Nix expressions to be parsed and evaluated, rather than as a list of file names of Nix expressions. + (`nix-instantiate`, `nix-build` and `nix-shell` only.) + + For `nix-shell`, this option is commonly used to give you a shell in which you can build the packages returned by the expression. + If you want to get a shell which contain the *built* packages ready for use, give your expression to the `nix-shell --packages ` convenience flag instead. + +- [`-I`](#opt-I) *path* + + Add an entry to the [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path). + This option may be given multiple times. + Paths added through `-I` take precedence over [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH). + +- [`--option`](#opt-option) *name* *value* + + Set the Nix configuration option *name* to *value*. + This overrides settings in the Nix configuration file (see nix.conf5). + +- [`--repair`](#opt-repair) + + Fix corrupted or missing store paths by redownloading or rebuilding them. + Note that this is slow because it requires computing a cryptographic hash of the contents of every path in the closure of the build. + Also note the warning under `nix-store --repair-path`. From 391f18063c0279e2de189176335a1b2863b9533a Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 6 Sep 2023 09:52:56 +0200 Subject: [PATCH 061/139] add anchors to option listings --- doc/manual/generate-manpage.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 65eec42d0..bc4d2c5bc 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -98,7 +98,7 @@ let (option ? labels) (concatStringsSep " " (map (s: "*${s}*") option.labels)); in trim '' - - `--${name}` ${shortName} ${labels} + - [`--${name}`](#opt-${name}) ${shortName} ${labels} ${option.description} ''; From d568877eab59a490694e0bc3edec99d13049552c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 6 Sep 2023 08:43:16 -0400 Subject: [PATCH 062/139] Retitle section as Robert suggests Co-authored-by: Robert Hensing --- doc/manual/src/SUMMARY.md.in | 2 +- doc/manual/src/protocols/derivation-aterm.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 3de053a17..701a6ea69 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -100,7 +100,7 @@ - [File System Object](architecture/file-system-object.md) - [Protocols](protocols/protocols.md) - [Serving Tarball Flakes](protocols/tarball-fetcher.md) - - [Derivation on-disk "ATerm" format](protocols/derivation-aterm.md) + - [Derivation "ATerm" file format](protocols/derivation-aterm.md) - [Glossary](glossary.md) - [Contributing](contributing/contributing.md) - [Hacking](contributing/hacking.md) diff --git a/doc/manual/src/protocols/derivation-aterm.md b/doc/manual/src/protocols/derivation-aterm.md index 2377e51ae..c9dc00ef7 100644 --- a/doc/manual/src/protocols/derivation-aterm.md +++ b/doc/manual/src/protocols/derivation-aterm.md @@ -1,4 +1,4 @@ -# Derivation on-disk "ATerm" format +# Derivation "ATerm" file format For historical reasons, [derivations](@docroot@/glossary.md#gloss-store-derivation) are stored on-disk in [ATerm](https://homepages.cwi.nl/~daybuild/daily-books/technology/aterm-guide/aterm-guide.html) format. From b7edc2099fffd7d54638122d2d9950e3d3a751f6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 6 Sep 2023 11:35:01 -0400 Subject: [PATCH 063/139] Improve derivation parsing - Don't assert: Derivation ATerms are not necessarily produced by Nix, and parsers should always throw graceful errors - Improve error message from `static void except(..)`, shows both what we expected and what we actually got. The intention is that we backport it, and then hopefully a few people might get slightly better errors if they try out new experimental drv files (for RFC 92) with an old version of Nix. --- src/libstore/derivations.cc | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index dc32c3847..43d3dc4a2 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -154,8 +154,9 @@ static void expect(std::istream & str, std::string_view s) { char s2[s.size()]; str.read(s2, s.size()); - if (std::string(s2, s.size()) != s) - throw FormatError("expected string '%1%'", s); + std::string_view s2View { s2, s.size() }; + if (s2View != s) + throw FormatError("expected string '%s', got '%s'", s, s2View); } @@ -223,7 +224,8 @@ static DerivationOutput parseDerivationOutput(const Store & store, const auto hashType = parseHashType(hashAlgo); if (hashS == "impure") { experimentalFeatureSettings.require(Xp::ImpureDerivations); - assert(pathS == ""); + if (pathS != "") + throw FormatError("impure derivation output should not specify output path"); return DerivationOutput::Impure { .method = std::move(method), .hashType = std::move(hashType), @@ -239,7 +241,8 @@ static DerivationOutput parseDerivationOutput(const Store & store, }; } else { experimentalFeatureSettings.require(Xp::CaDerivations); - assert(pathS == ""); + if (pathS != "") + throw FormatError("content-addressed derivation output should not specify output path"); return DerivationOutput::CAFloating { .method = std::move(method), .hashType = std::move(hashType), From 589fd897fb8414561b7b328e864e998876ff822b Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 6 Sep 2023 09:14:56 +0200 Subject: [PATCH 064/139] add todo on store docs --- doc/manual/generate-manpage.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index bc4d2c5bc..0f91a9742 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -70,6 +70,7 @@ let * [`${command} ${name}`](./${appendName filename name}.md) - ${subcmd.description} ''; + # TODO: move this confusing special case out of here when implementing #8496 maybeDocumentation = optionalString (details ? doc) (replaceStrings ["@stores@"] [storeDocs] details.doc); From b0fe7f560ddb31fea55546a79a89c44a63cd56ce Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 6 Sep 2023 09:15:03 +0200 Subject: [PATCH 065/139] add missing link --- doc/manual/generate-manpage.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 0f91a9742..705fc0b69 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -148,7 +148,7 @@ let To use this store, you need to make sure the corresponding experimental feature, [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}), is enabled. - For example, include the following in [`nix.conf`](#): + For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): ``` extra-experimental-features = ${experimentalFeature} From 09eb7f1ef653822616e7371c906ef1ae34b3ba09 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 6 Sep 2023 09:30:32 +0200 Subject: [PATCH 066/139] do not show configuration override flags for each command this removes a lot of noise from the web search, which precludes finding the actual documentation. some configuration settings have enough documentation to warrant individual pages, so the alternative of including full setting documentation in each command page doesn't make much sense here. this change technically means that the command line flags to override settings are "invisible", and not exported as JSON. this may or may not be desirable. a more explicit approach would be adding a `hidden` field to the flag's JSON output, but would also require adjusting post-processing of that JSON for manual rendering. --- doc/manual/generate-manpage.nix | 4 ++++ src/libutil/args.cc | 1 + 2 files changed, 5 insertions(+) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 705fc0b69..87891fa7a 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -79,6 +79,10 @@ let # Options ${showOptions details.flags toplevel.flags} + + > **Note** + > + > See [`man nix.conf`](@docroot@/command-ref/conf-file.md#command-line-flags) for overriding configuration settings with command line flags. ''; showOptions = options: commonOptions: diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 00281ce6f..8db293762 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -236,6 +236,7 @@ nlohmann::json Args::toJSON() for (auto & [name, flag] : longFlags) { auto j = nlohmann::json::object(); + if (hiddenCategories.count(flag->category)) continue; if (flag->aliases.count(name)) continue; if (flag->shortName) j["shortName"] = std::string(1, flag->shortName); From aa46f536e80ca20e0fc45664af867b47597a87f3 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 6 Sep 2023 09:40:51 +0200 Subject: [PATCH 067/139] add note on overriding settings for stable commands --- doc/manual/src/command-ref/opt-common.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index 0647228c2..114b292f9 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -203,3 +203,7 @@ Most Nix commands accept the following command-line options: Fix corrupted or missing store paths by redownloading or rebuilding them. Note that this is slow because it requires computing a cryptographic hash of the contents of every path in the closure of the build. Also note the warning under `nix-store --repair-path`. + +> **Note** +> +> See [`man nix.conf`](@docroot@/command-ref/conf-file.md#command-line-flags) for overriding configuration settings with command line flags. From 829d4d3e03e0f7935c1fd3e1ef760980c8cf6046 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 7 Sep 2023 15:13:22 +0200 Subject: [PATCH 068/139] fix invalid anchor link --- src/libexpr/eval-settings.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index e6666061a..19122bc31 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -21,7 +21,7 @@ struct EvalSettings : Config R"( List of directories to be searched for `<...>` file references - In particular, outside of [pure evaluation mode](#conf-pure-evaluation), this determines the value of + In particular, outside of [pure evaluation mode](#conf-pure-eval), this determines the value of [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath). )"}; From 7ad66cb3ef7615b51a35a099e232640d528c006c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 1 Oct 2021 18:05:53 -0400 Subject: [PATCH 069/139] Allow dynamic derivation deps in `inputDrvs` We use the same nested map representation we used for goals, again in order to save space. We might someday want to combine with `inputDrvs`, by doing `V = bool` instead of `V = std::set`, but we are not doing that yet for sake of a smaller diff. The ATerm format for Derivations also needs to be extended, in addition to the in-memory format. To accomodate this, we added a new basic versioning scheme, so old versions of Nix will get nice errors. (And going forward, if the ATerm format changes again the errors will be even better.) `parsedStrings`, an internal function used as part of parsing derivations in A-Term format, used to consume the final `]` but expect the initial `[` to already be consumed. This made for what looked like unbalanced brackets at callsites, which was confusing. Now it consumes both which is hopefully less confusing. As part of testing, we also created a unit test for the A-Term format for regular non-experimental derivations too. Co-authored-by: Robert Hensing Co-authored-by: Valentin Gagarin Apply suggestions from code review Co-authored-by: Robert Hensing --- doc/manual/src/protocols/derivation-aterm.md | 18 +- perl/lib/Nix/Store.xs | 2 +- src/build-remote/build-remote.cc | 2 +- src/libcmd/built-path.cc | 22 ++ src/libcmd/built-path.hh | 4 + src/libexpr/primops.cc | 10 +- src/libstore/build/derivation-goal.cc | 83 +++-- src/libstore/build/worker.cc | 9 +- src/libstore/build/worker.hh | 3 +- src/libstore/derivations.cc | 331 +++++++++++++++---- src/libstore/derivations.hh | 15 +- src/libstore/derived-path-map.cc | 41 ++- src/libstore/derived-path-map.hh | 25 ++ src/libstore/misc.cc | 67 ++-- src/libstore/tests/derivation.cc | 163 +++++++-- src/libutil/comparator.hh | 77 +++-- src/nix-build/nix-build.cc | 37 ++- src/nix/app.cc | 19 +- tests/dyn-drv/dep-built-drv.sh | 4 +- 19 files changed, 741 insertions(+), 191 deletions(-) diff --git a/doc/manual/src/protocols/derivation-aterm.md b/doc/manual/src/protocols/derivation-aterm.md index c9dc00ef7..e58b602a3 100644 --- a/doc/manual/src/protocols/derivation-aterm.md +++ b/doc/manual/src/protocols/derivation-aterm.md @@ -2,8 +2,18 @@ For historical reasons, [derivations](@docroot@/glossary.md#gloss-store-derivation) are stored on-disk in [ATerm](https://homepages.cwi.nl/~daybuild/daily-books/technology/aterm-guide/aterm-guide.html) format. -Derivations are serialised in the following format: +Derivations are serialised in one of the following formats: -``` -Derive(...) -``` +- ``` + Derive(...) + ``` + + For all stable derivations. + +- ``` + DrvWithVersion(, ...) + ``` + + The only `version-string`s that are in use today are for [experimental features](@docroot@/contributing/experimental-features.md): + + - `"xp-dyn-drv"` for the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index c38ea2d2b..e96885e4c 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -324,7 +324,7 @@ SV * derivationFromPath(char * drvPath) hv_stores(hash, "outputs", newRV((SV *) outputs)); AV * inputDrvs = newAV(); - for (auto & i : drv.inputDrvs) + for (auto & i : drv.inputDrvs.map) av_push(inputDrvs, newSVpv(store()->printStorePath(i.first).c_str(), 0)); // !!! ignores i->second hv_stores(hash, "inputDrvs", newRV((SV *) inputDrvs)); diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index c1f03a8ef..d69d3a0c2 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -314,7 +314,7 @@ connected: // // 2. Changing the `inputSrcs` set changes the associated // output ids, which break CA derivations - if (!drv.inputDrvs.empty()) + if (!drv.inputDrvs.map.empty()) drv.inputSrcs = store->parseStorePathSet(inputs); optResult = sshStore->buildDerivation(*drvPath, (const BasicDerivation &) drv); auto & result = *optResult; diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index c6cc7fa9c..9a2dce806 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -58,6 +58,28 @@ StorePathSet BuiltPath::outPaths() const ); } +SingleDerivedPath::Built SingleBuiltPath::Built::discardOutputPath() const +{ + return SingleDerivedPath::Built { + .drvPath = make_ref(drvPath->discardOutputPath()), + .output = output.first, + }; +} + +SingleDerivedPath SingleBuiltPath::discardOutputPath() const +{ + return std::visit( + overloaded{ + [](const SingleBuiltPath::Opaque & p) -> SingleDerivedPath { + return p; + }, + [](const SingleBuiltPath::Built & b) -> SingleDerivedPath { + return b.discardOutputPath(); + }, + }, raw() + ); +} + nlohmann::json BuiltPath::Built::toJSON(const Store & store) const { nlohmann::json res; diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh index 747bcc440..e677bc810 100644 --- a/src/libcmd/built-path.hh +++ b/src/libcmd/built-path.hh @@ -9,6 +9,8 @@ struct SingleBuiltPathBuilt { ref drvPath; std::pair output; + SingleDerivedPathBuilt discardOutputPath() const; + std::string to_string(const Store & store) const; static SingleBuiltPathBuilt parse(const Store & store, std::string_view, std::string_view); nlohmann::json toJSON(const Store & store) const; @@ -34,6 +36,8 @@ struct SingleBuiltPath : _SingleBuiltPathRaw { StorePath outPath() const; + SingleDerivedPath discardOutputPath() const; + static SingleBuiltPath parse(const Store & store, std::string_view); nlohmann::json toJSON(const Store & store) const; }; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index e2b1ac4f6..cd9a05bb2 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1252,15 +1252,13 @@ drvName, Bindings * attrs, Value & v) state.store->computeFSClosure(d.drvPath, refs); for (auto & j : refs) { drv.inputSrcs.insert(j); - if (j.isDerivation()) - drv.inputDrvs[j] = state.store->readDerivation(j).outputNames(); + if (j.isDerivation()) { + drv.inputDrvs.map[j].value = state.store->readDerivation(j).outputNames(); + } } }, [&](const NixStringContextElem::Built & b) { - if (auto * p = std::get_if(&*b.drvPath)) - drv.inputDrvs[p->path].insert(b.output); - else - throw UnimplementedError("Dependencies on the outputs of dynamic derivations are not yet supported"); + drv.inputDrvs.ensureSlot(*b.drvPath).value.insert(b.output); }, [&](const NixStringContextElem::Opaque & o) { drv.inputSrcs.insert(o.path); diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index bec0bc538..6472ecd99 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -350,25 +350,37 @@ void DerivationGoal::gaveUpOnSubstitution() /* The inputs must be built before we can build this goal. */ inputDrvOutputs.clear(); - if (useDerivation) - for (auto & i : dynamic_cast(drv.get())->inputDrvs) { + if (useDerivation) { + std::function, const DerivedPathMap::ChildNode &)> addWaiteeDerivedPath; + + addWaiteeDerivedPath = [&](ref inputDrv, const DerivedPathMap::ChildNode & inputNode) { + if (!inputNode.value.empty()) + addWaitee(worker.makeGoal( + DerivedPath::Built { + .drvPath = inputDrv, + .outputs = inputNode.value, + }, + buildMode == bmRepair ? bmRepair : bmNormal)); + for (const auto & [outputName, childNode] : inputNode.childMap) + addWaiteeDerivedPath( + make_ref(SingleDerivedPath::Built { inputDrv, outputName }), + childNode); + }; + + for (const auto & [inputDrvPath, inputNode] : dynamic_cast(drv.get())->inputDrvs.map) { /* Ensure that pure, non-fixed-output derivations don't depend on impure derivations. */ if (experimentalFeatureSettings.isEnabled(Xp::ImpureDerivations) && drv->type().isPure() && !drv->type().isFixed()) { - auto inputDrv = worker.evalStore.readDerivation(i.first); + auto inputDrv = worker.evalStore.readDerivation(inputDrvPath); if (!inputDrv.type().isPure()) throw Error("pure derivation '%s' depends on impure derivation '%s'", worker.store.printStorePath(drvPath), - worker.store.printStorePath(i.first)); + worker.store.printStorePath(inputDrvPath)); } - addWaitee(worker.makeGoal( - DerivedPath::Built { - .drvPath = makeConstantStorePathRef(i.first), - .outputs = i.second, - }, - buildMode == bmRepair ? bmRepair : bmNormal)); + addWaiteeDerivedPath(makeConstantStorePathRef(inputDrvPath), inputNode); } + } /* Copy the input sources from the eval store to the build store. */ @@ -501,7 +513,7 @@ void DerivationGoal::inputsRealised() return ia.deferred; }, [&](const DerivationType::ContentAddressed & ca) { - return !fullDrv.inputDrvs.empty() && ( + return !fullDrv.inputDrvs.map.empty() && ( ca.fixed /* Can optionally resolve if fixed, which is good for avoiding unnecessary rebuilds. */ @@ -515,7 +527,7 @@ void DerivationGoal::inputsRealised() } }, drvType.raw); - if (resolveDrv && !fullDrv.inputDrvs.empty()) { + if (resolveDrv && !fullDrv.inputDrvs.map.empty()) { experimentalFeatureSettings.require(Xp::CaDerivations); /* We are be able to resolve this derivation based on the @@ -552,11 +564,13 @@ void DerivationGoal::inputsRealised() return; } - for (auto & [depDrvPath, wantedDepOutputs] : fullDrv.inputDrvs) { + std::function::ChildNode &)> accumInputPaths; + + accumInputPaths = [&](const StorePath & depDrvPath, const DerivedPathMap::ChildNode & inputNode) { /* Add the relevant output closures of the input derivation `i' as input paths. Only add the closures of output paths that are specified as inputs. */ - for (auto & j : wantedDepOutputs) { + auto getOutput = [&](const std::string & outputName) { /* TODO (impure derivations-induced tech debt): Tracking input derivation outputs statefully through the goals is error prone and has led to bugs. @@ -568,21 +582,30 @@ void DerivationGoal::inputsRealised() a representation in the store, which is a usability problem in itself. When implementing this logic entirely with lookups make sure that they're cached. */ - if (auto outPath = get(inputDrvOutputs, { depDrvPath, j })) { - worker.store.computeFSClosure(*outPath, inputPaths); + if (auto outPath = get(inputDrvOutputs, { depDrvPath, outputName })) { + return *outPath; } else { auto outMap = worker.evalStore.queryDerivationOutputMap(depDrvPath); - auto outMapPath = outMap.find(j); + auto outMapPath = outMap.find(outputName); if (outMapPath == outMap.end()) { throw Error( "derivation '%s' requires non-existent output '%s' from input derivation '%s'", - worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath)); + worker.store.printStorePath(drvPath), outputName, worker.store.printStorePath(depDrvPath)); } - worker.store.computeFSClosure(outMapPath->second, inputPaths); + return outMapPath->second; } - } - } + }; + + for (auto & outputName : inputNode.value) + worker.store.computeFSClosure(getOutput(outputName), inputPaths); + + for (auto & [outputName, childNode] : inputNode.childMap) + accumInputPaths(getOutput(outputName), childNode); + }; + + for (auto & [depDrvPath, depNode] : fullDrv.inputDrvs.map) + accumInputPaths(depDrvPath, depNode); } /* Second, the input sources. */ @@ -1475,22 +1498,24 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result) if (!useDerivation) return; auto & fullDrv = *dynamic_cast(drv.get()); - auto * dg = tryGetConcreteDrvGoal(waitee); - if (!dg) return; + std::optional info = tryGetConcreteDrvGoal(waitee); + if (!info) return; + const auto & [dg, drvReq] = *info; - auto outputs = fullDrv.inputDrvs.find(dg->drvPath); - if (outputs == fullDrv.inputDrvs.end()) return; + auto * nodeP = fullDrv.inputDrvs.findSlot(drvReq.get()); + if (!nodeP) return; + auto & outputs = nodeP->value; - for (auto & outputName : outputs->second) { - auto buildResult = dg->getBuildResult(DerivedPath::Built { - .drvPath = makeConstantStorePathRef(dg->drvPath), + for (auto & outputName : outputs) { + auto buildResult = dg.get().getBuildResult(DerivedPath::Built { + .drvPath = makeConstantStorePathRef(dg.get().drvPath), .outputs = OutputsSpec::Names { outputName }, }); if (buildResult.success()) { auto i = buildResult.builtOutputs.find(outputName); if (i != buildResult.builtOutputs.end()) inputDrvOutputs.insert_or_assign( - { dg->drvPath, outputName }, + { dg.get().drvPath, outputName }, i->second.outPath); } } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index f65f63b99..b4a634e7b 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -594,11 +594,14 @@ GoalPtr upcast_goal(std::shared_ptr subGoal) return subGoal; } -const DerivationGoal * tryGetConcreteDrvGoal(GoalPtr waitee) +std::optional, std::reference_wrapper>> tryGetConcreteDrvGoal(GoalPtr waitee) { auto * odg = dynamic_cast(&*waitee); - if (!odg) return nullptr; - return &*odg->concreteDrvGoal; + if (!odg) return std::nullopt; + return {{ + std::cref(*odg->concreteDrvGoal), + std::cref(*odg->drvReq), + }}; } } diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index a778e311c..6f6d25d7d 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -49,7 +49,8 @@ typedef std::chrono::time_point steady_time_point; * we have made the function, written in `worker.cc` where all the goal * types are visible, and use it instead. */ -const DerivationGoal * tryGetConcreteDrvGoal(GoalPtr waitee); + +std::optional, std::reference_wrapper>> tryGetConcreteDrvGoal(GoalPtr waitee); /** * A mapping used to remember for each child process to what goal it diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 43d3dc4a2..67069c3c9 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -136,7 +136,7 @@ StorePath writeDerivation(Store & store, const Derivation & drv, RepairFlag repair, bool readOnly) { auto references = drv.inputSrcs; - for (auto & i : drv.inputDrvs) + for (auto & i : drv.inputDrvs.map) references.insert(i.first); /* Note that the outputs of a derivation are *not* references (that can be missing (of course) and should not necessarily be @@ -208,22 +208,25 @@ static bool endOfList(std::istream & str) static StringSet parseStrings(std::istream & str, bool arePaths) { StringSet res; + expect(str, "["); while (!endOfList(str)) res.insert(arePaths ? parsePath(str) : parseString(str)); return res; } -static DerivationOutput parseDerivationOutput(const Store & store, - std::string_view pathS, std::string_view hashAlgo, std::string_view hashS) +static DerivationOutput parseDerivationOutput( + const Store & store, + std::string_view pathS, std::string_view hashAlgo, std::string_view hashS, + const ExperimentalFeatureSettings & xpSettings) { if (hashAlgo != "") { ContentAddressMethod method = ContentAddressMethod::parsePrefix(hashAlgo); if (method == TextIngestionMethod {}) - experimentalFeatureSettings.require(Xp::DynamicDerivations); + xpSettings.require(Xp::DynamicDerivations); const auto hashType = parseHashType(hashAlgo); if (hashS == "impure") { - experimentalFeatureSettings.require(Xp::ImpureDerivations); + xpSettings.require(Xp::ImpureDerivations); if (pathS != "") throw FormatError("impure derivation output should not specify output path"); return DerivationOutput::Impure { @@ -240,7 +243,7 @@ static DerivationOutput parseDerivationOutput(const Store & store, }, }; } else { - experimentalFeatureSettings.require(Xp::CaDerivations); + xpSettings.require(Xp::CaDerivations); if (pathS != "") throw FormatError("content-addressed derivation output should not specify output path"); return DerivationOutput::CAFloating { @@ -259,29 +262,116 @@ static DerivationOutput parseDerivationOutput(const Store & store, } } -static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str) +static DerivationOutput parseDerivationOutput( + const Store & store, std::istringstream & str, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings) { expect(str, ","); const auto pathS = parseString(str); expect(str, ","); const auto hashAlgo = parseString(str); expect(str, ","); const auto hash = parseString(str); expect(str, ")"); - return parseDerivationOutput(store, pathS, hashAlgo, hash); + return parseDerivationOutput(store, pathS, hashAlgo, hash, xpSettings); +} + +/** + * All ATerm Derivation format versions currently known. + * + * Unknown versions are rejected at the parsing stage. + */ +enum struct DerivationATermVersion { + /** + * Older unversioned form + */ + Traditional, + + /** + * Newer versioned form; only this version so far. + */ + DynamicDerivations, +}; + +static DerivedPathMap::ChildNode parseDerivedPathMapNode( + const Store & store, + std::istringstream & str, + DerivationATermVersion version) +{ + DerivedPathMap::ChildNode node; + + auto parseNonDynamic = [&]() { + node.value = parseStrings(str, false); + }; + + // Older derivation should never use new form, but newer + // derivaiton can use old form. + switch (version) { + case DerivationATermVersion::Traditional: + parseNonDynamic(); + break; + case DerivationATermVersion::DynamicDerivations: + switch (str.peek()) { + case '[': + parseNonDynamic(); + break; + case '(': + expect(str, "("); + node.value = parseStrings(str, false); + expect(str, ",["); + while (!endOfList(str)) { + expect(str, "("); + auto outputName = parseString(str); + expect(str, ","); + node.childMap.insert_or_assign(outputName, parseDerivedPathMapNode(store, str, version)); + expect(str, ")"); + } + expect(str, ")"); + break; + default: + throw FormatError("invalid inputDrvs entry in derivation"); + } + break; + default: + // invalid format, not a parse error but internal error + assert(false); + } + return node; } -Derivation parseDerivation(const Store & store, std::string && s, std::string_view name) +Derivation parseDerivation( + const Store & store, std::string && s, std::string_view name, + const ExperimentalFeatureSettings & xpSettings) { Derivation drv; drv.name = name; std::istringstream str(std::move(s)); - expect(str, "Derive(["); + expect(str, "D"); + DerivationATermVersion version; + switch (str.peek()) { + case 'e': + expect(str, "erive("); + version = DerivationATermVersion::Traditional; + break; + case 'r': + expect(str, "rvWithVersion("); + auto versionS = parseString(str); + if (versionS == "xp-dyn-drv") { + // Only verison we have so far + version = DerivationATermVersion::DynamicDerivations; + xpSettings.require(Xp::DynamicDerivations); + } else { + throw FormatError("Unknown derivation ATerm format version '%s'", versionS); + } + expect(str, ","); + break; + } /* Parse the list of outputs. */ + expect(str, "["); while (!endOfList(str)) { expect(str, "("); std::string id = parseString(str); - auto output = parseDerivationOutput(store, str); + auto output = parseDerivationOutput(store, str, xpSettings); drv.outputs.emplace(std::move(id), std::move(output)); } @@ -290,12 +380,12 @@ Derivation parseDerivation(const Store & store, std::string && s, std::string_vi while (!endOfList(str)) { expect(str, "("); Path drvPath = parsePath(str); - expect(str, ",["); - drv.inputDrvs.insert_or_assign(store.parseStorePath(drvPath), parseStrings(str, false)); + expect(str, ","); + drv.inputDrvs.map.insert_or_assign(store.parseStorePath(drvPath), parseDerivedPathMapNode(store, str, version)); expect(str, ")"); } - expect(str, ",["); drv.inputSrcs = store.parseStorePathSet(parseStrings(str, true)); + expect(str, ","); drv.inputSrcs = store.parseStorePathSet(parseStrings(str, true)); expect(str, ","); drv.platform = parseString(str); expect(str, ","); drv.builder = parseString(str); @@ -379,14 +469,67 @@ static void printUnquotedStrings(std::string & res, ForwardIterator i, ForwardIt } +static void unparseDerivedPathMapNode(const Store & store, std::string & s, const DerivedPathMap::ChildNode & node) +{ + s += ','; + if (node.childMap.empty()) { + printUnquotedStrings(s, node.value.begin(), node.value.end()); + } else { + s += "("; + printUnquotedStrings(s, node.value.begin(), node.value.end()); + s += ",["; + bool first = true; + for (auto & [outputName, childNode] : node.childMap) { + if (first) first = false; else s += ','; + s += '('; printUnquotedString(s, outputName); + unparseDerivedPathMapNode(store, s, childNode); + s += ')'; + } + s += "])"; + } +} + + +/** + * Does the derivation have a dependency on the output of a dynamic + * derivation? + * + * In other words, does it on the output of derivation that is itself an + * ouput of a derivation? This corresponds to a dependency that is an + * inductive derived path with more than one layer of + * `DerivedPath::Built`. + */ +static bool hasDynamicDrvDep(const Derivation & drv) +{ + return + std::find_if( + drv.inputDrvs.map.begin(), + drv.inputDrvs.map.end(), + [](auto & kv) { return !kv.second.childMap.empty(); }) + != drv.inputDrvs.map.end(); +} + + std::string Derivation::unparse(const Store & store, bool maskOutputs, - std::map * actualInputs) const + DerivedPathMap::ChildNode::Map * actualInputs) const { std::string s; s.reserve(65536); - s += "Derive(["; + + /* Use older unversioned form if possible, for wider compat. Use + newer form only if we need it, which we do for + `Xp::DynamicDerivations`. */ + if (hasDynamicDrvDep(*this)) { + s += "DrvWithVersion("; + // Only version we have so far + printUnquotedString(s, "xp-dyn-drv"); + s += ","; + } else { + s += "Derive("; + } bool first = true; + s += "["; for (auto & i : outputs) { if (first) first = false; else s += ','; s += '('; printUnquotedString(s, i.first); @@ -424,17 +567,17 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs, s += "],["; first = true; if (actualInputs) { - for (auto & i : *actualInputs) { + for (auto & [drvHashModulo, childMap] : *actualInputs) { if (first) first = false; else s += ','; - s += '('; printUnquotedString(s, i.first); - s += ','; printUnquotedStrings(s, i.second.begin(), i.second.end()); + s += '('; printUnquotedString(s, drvHashModulo); + unparseDerivedPathMapNode(store, s, childMap); s += ')'; } } else { - for (auto & i : inputDrvs) { + for (auto & [drvPath, childMap] : inputDrvs.map) { if (first) first = false; else s += ','; - s += '('; printUnquotedString(s, store.printStorePath(i.first)); - s += ','; printUnquotedStrings(s, i.second.begin(), i.second.end()); + s += '('; printUnquotedString(s, store.printStorePath(drvPath)); + unparseDerivedPathMapNode(store, s, childMap); s += ')'; } } @@ -668,18 +811,16 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut } }, drv.type().raw); - std::map inputs2; - for (auto & [drvPath, inputOutputs0] : drv.inputDrvs) { - // Avoid lambda capture restriction with standard / Clang - auto & inputOutputs = inputOutputs0; + DerivedPathMap::ChildNode::Map inputs2; + for (auto & [drvPath, node] : drv.inputDrvs.map) { const auto & res = pathDerivationModulo(store, drvPath); if (res.kind == DrvHash::Kind::Deferred) kind = DrvHash::Kind::Deferred; - for (auto & outputName : inputOutputs) { + for (auto & outputName : node.value) { const auto h = get(res.hashes, outputName); if (!h) throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name); - inputs2[h->to_string(Base16, false)].insert(outputName); + inputs2[h->to_string(Base16, false)].value.insert(outputName); } } @@ -709,7 +850,7 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store) const auto hashAlgo = readString(in); const auto hash = readString(in); - return parseDerivationOutput(store, pathS, hashAlgo, hash); + return parseDerivationOutput(store, pathS, hashAlgo, hash, experimentalFeatureSettings); } StringSet BasicDerivation::outputNames() const @@ -824,6 +965,8 @@ std::string hashPlaceholder(const OutputNameView outputName) static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) { + debug("Rewriting the derivation"); + for (auto & rewrite : rewrites) { debug("rewriting %s as %s", rewrite.first, rewrite.second); } @@ -862,14 +1005,70 @@ std::optional Derivation::tryResolve(Store & store) const { std::map, StorePath> inputDrvOutputs; - for (auto & input : inputDrvs) - for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(input.first)) - if (outputPath) - inputDrvOutputs.insert_or_assign({input.first, outputName}, *outputPath); + std::function::ChildNode &)> accum; + accum = [&](auto & inputDrv, auto & node) { + for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(inputDrv)) { + if (outputPath) { + inputDrvOutputs.insert_or_assign({inputDrv, outputName}, *outputPath); + if (auto p = get(node.childMap, outputName)) + accum(*outputPath, *p); + } + } + }; + + for (auto & [inputDrv, node] : inputDrvs.map) + accum(inputDrv, node); return tryResolve(store, inputDrvOutputs); } +static bool tryResolveInput( + Store & store, StorePathSet & inputSrcs, StringMap & inputRewrites, + const DownstreamPlaceholder * placeholderOpt, + const StorePath & inputDrv, const DerivedPathMap::ChildNode & inputNode, + const std::map, StorePath> & inputDrvOutputs) +{ + auto getOutput = [&](const std::string & outputName) { + auto * actualPathOpt = get(inputDrvOutputs, { inputDrv, outputName }); + if (!actualPathOpt) + warn("output %s of input %s missing, aborting the resolving", + outputName, + store.printStorePath(inputDrv) + ); + return actualPathOpt; + }; + + auto getPlaceholder = [&](const std::string & outputName) { + return placeholderOpt + ? DownstreamPlaceholder::unknownDerivation(*placeholderOpt, outputName) + : DownstreamPlaceholder::unknownCaOutput(inputDrv, outputName); + }; + + for (auto & outputName : inputNode.value) { + auto actualPathOpt = getOutput(outputName); + if (!actualPathOpt) return false; + auto actualPath = *actualPathOpt; + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { + inputRewrites.emplace( + getPlaceholder(outputName).render(), + store.printStorePath(actualPath)); + } + inputSrcs.insert(std::move(actualPath)); + } + + for (auto & [outputName, childNode] : inputNode.childMap) { + auto actualPathOpt = getOutput(outputName); + if (!actualPathOpt) return false; + auto actualPath = *actualPathOpt; + auto nextPlaceholder = getPlaceholder(outputName); + if (!tryResolveInput(store, inputSrcs, inputRewrites, + &nextPlaceholder, actualPath, childNode, + inputDrvOutputs)) + return false; + } + return true; +} + std::optional Derivation::tryResolve( Store & store, const std::map, StorePath> & inputDrvOutputs) const @@ -879,23 +1078,10 @@ std::optional Derivation::tryResolve( // Input paths that we'll want to rewrite in the derivation StringMap inputRewrites; - for (auto & [inputDrv, inputOutputs] : inputDrvs) { - for (auto & outputName : inputOutputs) { - if (auto actualPath = get(inputDrvOutputs, { inputDrv, outputName })) { - if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { - inputRewrites.emplace( - DownstreamPlaceholder::unknownCaOutput(inputDrv, outputName).render(), - store.printStorePath(*actualPath)); - } - resolved.inputSrcs.insert(*actualPath); - } else { - warn("output '%s' of input '%s' missing, aborting the resolving", - outputName, - store.printStorePath(inputDrv)); - return {}; - } - } - } + for (auto & [inputDrv, inputNode] : inputDrvs.map) + if (!tryResolveInput(store, resolved.inputSrcs, inputRewrites, + nullptr, inputDrv, inputNode, inputDrvOutputs)) + return std::nullopt; rewriteDerivation(store, resolved, inputRewrites); @@ -1084,10 +1270,25 @@ nlohmann::json Derivation::toJSON(const Store & store) const } { - auto& inputDrvsObj = res["inputDrvs"]; - inputDrvsObj = nlohmann::json ::object(); - for (auto & input : inputDrvs) - inputDrvsObj[store.printStorePath(input.first)] = input.second; + std::function::ChildNode &)> doInput; + doInput = [&](const auto & inputNode) { + auto value = nlohmann::json::object(); + value["outputs"] = inputNode.value; + { + auto next = nlohmann::json::object(); + for (auto & [outputId, childNode] : inputNode.childMap) + next[outputId] = doInput(childNode); + value["dynamicOutputs"] = std::move(next); + } + return value; + }; + { + auto& inputDrvsObj = res["inputDrvs"]; + inputDrvsObj = nlohmann::json::object(); + for (auto & [inputDrv, inputNode] : inputDrvs.map) { + inputDrvsObj[store.printStorePath(inputDrv)] = doInput(inputNode); + } + } } res["system"] = platform; @@ -1101,7 +1302,8 @@ nlohmann::json Derivation::toJSON(const Store & store) const Derivation Derivation::fromJSON( const Store & store, - const nlohmann::json & json) + const nlohmann::json & json, + const ExperimentalFeatureSettings & xpSettings) { using nlohmann::detail::value_t; @@ -1133,12 +1335,21 @@ Derivation Derivation::fromJSON( } try { + std::function::ChildNode(const nlohmann::json &)> doInput; + doInput = [&](const auto & json) { + DerivedPathMap::ChildNode node; + node.value = static_cast( + ensureType(valueAt(json, "outputs"), value_t::array)); + for (auto & [outputId, childNode] : ensureType(valueAt(json, "dynamicOutputs"), value_t::object).items()) { + xpSettings.require(Xp::DynamicDerivations); + node.childMap[outputId] = doInput(childNode); + } + return node; + }; auto & inputDrvsObj = ensureType(valueAt(json, "inputDrvs"), value_t::object); - for (auto & [inputDrvPath, inputOutputs] : inputDrvsObj.items()) { - ensureType(inputOutputs, value_t::array); - res.inputDrvs[store.parseStorePath(inputDrvPath)] = - static_cast(inputOutputs); - } + for (auto & [inputDrvPath, inputOutputs] : inputDrvsObj.items()) + res.inputDrvs.map[store.parseStorePath(inputDrvPath)] = + doInput(inputOutputs); } catch (Error & e) { e.addTrace({}, "while reading key 'inputDrvs'"); throw; diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 106056f2d..fa14e7536 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -6,7 +6,7 @@ #include "hash.hh" #include "content-address.hh" #include "repair-flag.hh" -#include "derived-path.hh" +#include "derived-path-map.hh" #include "sync.hh" #include "comparator.hh" #include "variant-wrapper.hh" @@ -323,13 +323,13 @@ struct Derivation : BasicDerivation /** * inputs that are sub-derivations */ - DerivationInputs inputDrvs; + DerivedPathMap> inputDrvs; /** * Print a derivation. */ std::string unparse(const Store & store, bool maskOutputs, - std::map * actualInputs = nullptr) const; + DerivedPathMap::ChildNode::Map * actualInputs = nullptr) const; /** * Return the underlying basic derivation but with these changes: @@ -368,7 +368,8 @@ struct Derivation : BasicDerivation nlohmann::json toJSON(const Store & store) const; static Derivation fromJSON( const Store & store, - const nlohmann::json & json); + const nlohmann::json & json, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); GENERATE_CMP(Derivation, static_cast(*me), @@ -389,7 +390,11 @@ StorePath writeDerivation(Store & store, /** * Read a derivation from a file. */ -Derivation parseDerivation(const Store & store, std::string && s, std::string_view name); +Derivation parseDerivation( + const Store & store, + std::string && s, + std::string_view name, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); /** * \todo Remove. diff --git a/src/libstore/derived-path-map.cc b/src/libstore/derived-path-map.cc index 5c8c7a4f2..437b6a71a 100644 --- a/src/libstore/derived-path-map.cc +++ b/src/libstore/derived-path-map.cc @@ -21,6 +21,32 @@ typename DerivedPathMap::ChildNode & DerivedPathMap::ensureSlot(const Sing return initIter(k); } +template +typename DerivedPathMap::ChildNode * DerivedPathMap::findSlot(const SingleDerivedPath & k) +{ + std::function initIter; + initIter = [&](const auto & k) { + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) { + auto it = map.find(bo.path); + return it != map.end() + ? &it->second + : nullptr; + }, + [&](const SingleDerivedPath::Built & bfd) { + auto * n = initIter(*bfd.drvPath); + if (!n) return (ChildNode *)nullptr; + + auto it = n->childMap.find(bfd.output); + return it != n->childMap.end() + ? &it->second + : nullptr; + }, + }, k.raw()); + }; + return initIter(k); +} + } // instantiations @@ -30,4 +56,17 @@ namespace nix { template struct DerivedPathMap>; -} +GENERATE_CMP_EXT( + template<>, + DerivedPathMap>::ChildNode, + me->value, + me->childMap); + +GENERATE_CMP_EXT( + template<>, + DerivedPathMap>, + me->map); + +template struct DerivedPathMap>; + +}; diff --git a/src/libstore/derived-path-map.hh b/src/libstore/derived-path-map.hh index 9ce58206e..4a2c90733 100644 --- a/src/libstore/derived-path-map.hh +++ b/src/libstore/derived-path-map.hh @@ -48,6 +48,8 @@ struct DerivedPathMap { * The map of the root node. */ Map childMap; + + DECLARE_CMP(ChildNode); }; /** @@ -60,6 +62,8 @@ struct DerivedPathMap { */ Map map; + DECLARE_CMP(DerivedPathMap); + /** * Find the node for `k`, creating it if needed. * @@ -68,6 +72,27 @@ struct DerivedPathMap { * by changing this node. */ ChildNode & ensureSlot(const SingleDerivedPath & k); + + /** + * Like `ensureSlot` but does not create the slot if it doesn't exist. + * + * Read the entire description of `ensureSlot` to understand an + * important caveat here that "have slot" does *not* imply "key is + * set in map". To ensure a key is set one would need to get the + * child node (with `findSlot` or `ensureSlot`) *and* check the + * `ChildNode::value`. + */ + ChildNode * findSlot(const SingleDerivedPath & k); }; + +DECLARE_CMP_EXT( + template<>, + DerivedPathMap>::, + DerivedPathMap>); +DECLARE_CMP_EXT( + template<>, + DerivedPathMap>::ChildNode::, + DerivedPathMap>::ChildNode); + } diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index c043b9b93..1035691c7 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -125,14 +125,26 @@ void Store::queryMissing(const std::vector & targets, std::function doPath; + std::function, const DerivedPathMap::ChildNode &)> enqueueDerivedPaths; + + enqueueDerivedPaths = [&](ref inputDrv, const DerivedPathMap::ChildNode & inputNode) { + if (!inputNode.value.empty()) + pool.enqueue(std::bind(doPath, DerivedPath::Built { inputDrv, inputNode.value })); + for (const auto & [outputName, childNode] : inputNode.childMap) + enqueueDerivedPaths( + make_ref(SingleDerivedPath::Built { inputDrv, outputName }), + childNode); + }; + auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv) { { auto state(state_.lock()); state->willBuild.insert(drvPath); } - for (auto & i : drv.inputDrvs) - pool.enqueue(std::bind(doPath, DerivedPath::Built { makeConstantStorePathRef(i.first), i.second })); + for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map) { + enqueueDerivedPaths(makeConstantStorePathRef(inputDrv), inputNode); + } }; auto checkOutput = [&]( @@ -322,24 +334,41 @@ std::map drvOutputReferences( { std::set inputRealisations; - for (const auto & [inputDrv, outputNames] : drv.inputDrvs) { - const auto outputHashes = - staticOutputHashes(store, store.readDerivation(inputDrv)); - for (const auto & outputName : outputNames) { - auto outputHash = get(outputHashes, outputName); - if (!outputHash) - throw Error( - "output '%s' of derivation '%s' isn't realised", outputName, - store.printStorePath(inputDrv)); - auto thisRealisation = store.queryRealisation( - DrvOutput{*outputHash, outputName}); - if (!thisRealisation) - throw Error( - "output '%s' of derivation '%s' isn't built", outputName, - store.printStorePath(inputDrv)); - inputRealisations.insert(*thisRealisation); + std::function::ChildNode &)> accumRealisations; + + accumRealisations = [&](const StorePath & inputDrv, const DerivedPathMap::ChildNode & inputNode) { + if (!inputNode.value.empty()) { + auto outputHashes = + staticOutputHashes(store, store.readDerivation(inputDrv)); + for (const auto & outputName : inputNode.value) { + auto outputHash = get(outputHashes, outputName); + if (!outputHash) + throw Error( + "output '%s' of derivation '%s' isn't realised", outputName, + store.printStorePath(inputDrv)); + auto thisRealisation = store.queryRealisation( + DrvOutput{*outputHash, outputName}); + if (!thisRealisation) + throw Error( + "output '%s' of derivation '%s' isn’t built", outputName, + store.printStorePath(inputDrv)); + inputRealisations.insert(*thisRealisation); + } } - } + if (!inputNode.value.empty()) { + auto d = makeConstantStorePathRef(inputDrv); + for (const auto & [outputName, childNode] : inputNode.childMap) { + SingleDerivedPath next = SingleDerivedPath::Built { d, outputName }; + accumRealisations( + // TODO deep resolutions for dynamic derivations, issue #8947, would go here. + resolveDerivedPath(store, next), + childNode); + } + } + }; + + for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map) + accumRealisations(inputDrv, inputNode); auto info = store.queryPathInfo(outputPath); diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc index 7fd7f9278..c360c9707 100644 --- a/src/libstore/tests/derivation.cc +++ b/src/libstore/tests/derivation.cc @@ -42,6 +42,26 @@ class ImpureDerivationTest : public DerivationTest } }; +TEST_F(DerivationTest, BadATerm_version) { + ASSERT_THROW( + parseDerivation( + *store, + R"(DrvWithVersion("invalid-version",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))", + "whatever", + mockXpSettings), + FormatError); +} + +TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps) { + ASSERT_THROW( + parseDerivation( + *store, + R"(Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))", + "dyn-dep-derivation", + mockXpSettings), + FormatError); +} + #define TEST_JSON(FIXTURE, NAME, STR, VAL, DRV_NAME, OUTPUT_NAME) \ TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _to_json) { \ using nlohmann::literals::operator "" _json; \ @@ -143,35 +163,37 @@ TEST_JSON(ImpureDerivationTest, impure, #undef TEST_JSON -#define TEST_JSON(NAME, STR, VAL) \ - TEST_F(DerivationTest, Derivation_ ## NAME ## _to_json) { \ +#define TEST_JSON(FIXTURE, NAME, STR, VAL) \ + TEST_F(FIXTURE, Derivation_ ## NAME ## _to_json) { \ using nlohmann::literals::operator "" _json; \ ASSERT_EQ( \ STR ## _json, \ (VAL).toJSON(*store)); \ } \ \ - TEST_F(DerivationTest, Derivation_ ## NAME ## _from_json) { \ + TEST_F(FIXTURE, Derivation_ ## NAME ## _from_json) { \ using nlohmann::literals::operator "" _json; \ ASSERT_EQ( \ (VAL), \ Derivation::fromJSON( \ *store, \ - STR ## _json)); \ + STR ## _json, \ + mockXpSettings)); \ } -#define TEST_ATERM(NAME, STR, VAL, DRV_NAME) \ - TEST_F(DerivationTest, Derivation_ ## NAME ## _to_aterm) { \ +#define TEST_ATERM(FIXTURE, NAME, STR, VAL, DRV_NAME) \ + TEST_F(FIXTURE, Derivation_ ## NAME ## _to_aterm) { \ ASSERT_EQ( \ STR, \ (VAL).unparse(*store, false)); \ } \ \ - TEST_F(DerivationTest, Derivation_ ## NAME ## _from_aterm) { \ + TEST_F(FIXTURE, Derivation_ ## NAME ## _from_aterm) { \ auto parsed = parseDerivation( \ - *store, \ - STR, \ - DRV_NAME); \ + *store, \ + STR, \ + DRV_NAME, \ + mockXpSettings); \ ASSERT_EQ( \ (VAL).toJSON(*store), \ parsed.toJSON(*store)); \ @@ -187,11 +209,15 @@ Derivation makeSimpleDrv(const Store & store) { store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), }; drv.inputDrvs = { - { - store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), + .map = { { - "cat", - "dog", + store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), + { + .value = { + "cat", + "dog", + }, + }, }, }, }; @@ -210,17 +236,20 @@ Derivation makeSimpleDrv(const Store & store) { return drv; } -TEST_JSON(simple, +TEST_JSON(DerivationTest, simple, R"({ "name": "simple-derivation", "inputSrcs": [ "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" ], "inputDrvs": { - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": [ - "cat", - "dog" - ] + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { + "dynamicOutputs": {}, + "outputs": [ + "cat", + "dog" + ] + } }, "system": "wasm-sel4", "builder": "foo", @@ -235,11 +264,105 @@ TEST_JSON(simple, })", makeSimpleDrv(*store)) -TEST_ATERM(simple, +TEST_ATERM(DerivationTest, simple, R"(Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))", makeSimpleDrv(*store), "simple-derivation") +Derivation makeDynDepDerivation(const Store & store) { + Derivation drv; + drv.name = "dyn-dep-derivation"; + drv.inputSrcs = { + store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), + }; + drv.inputDrvs = { + .map = { + { + store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), + DerivedPathMap::ChildNode { + .value = { + "cat", + "dog", + }, + .childMap = { + { + "cat", + DerivedPathMap::ChildNode { + .value = { + "kitten", + }, + }, + }, + { + "goose", + DerivedPathMap::ChildNode { + .value = { + "gosling", + }, + }, + }, + }, + }, + }, + }, + }; + drv.platform = "wasm-sel4"; + drv.builder = "foo"; + drv.args = { + "bar", + "baz", + }; + drv.env = { + { + "BIG_BAD", + "WOLF", + }, + }; + return drv; +} + +TEST_JSON(DynDerivationTest, dynDerivationDeps, + R"({ + "name": "dyn-dep-derivation", + "inputSrcs": [ + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" + ], + "inputDrvs": { + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { + "dynamicOutputs": { + "cat": { + "dynamicOutputs": {}, + "outputs": ["kitten"] + }, + "goose": { + "dynamicOutputs": {}, + "outputs": ["gosling"] + } + }, + "outputs": [ + "cat", + "dog" + ] + } + }, + "system": "wasm-sel4", + "builder": "foo", + "args": [ + "bar", + "baz" + ], + "env": { + "BIG_BAD": "WOLF" + }, + "outputs": {} + })", + makeDynDepDerivation(*store)) + +TEST_ATERM(DynDerivationTest, dynDerivationDeps, + R"(DrvWithVersion("xp-dyn-drv",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))", + makeDynDepDerivation(*store), + "dyn-dep-derivation") + #undef TEST_JSON #undef TEST_ATERM diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh index 7982fdc5e..a4d20a675 100644 --- a/src/libutil/comparator.hh +++ b/src/libutil/comparator.hh @@ -1,21 +1,48 @@ #pragma once ///@file +#define DECLARE_ONE_CMP(PRE, QUAL, COMPARATOR, MY_TYPE) \ + PRE bool QUAL operator COMPARATOR(const MY_TYPE & other) const; +#define DECLARE_EQUAL(prefix, qualification, my_type) \ + DECLARE_ONE_CMP(prefix, qualification, ==, my_type) +#define DECLARE_LEQ(prefix, qualification, my_type) \ + DECLARE_ONE_CMP(prefix, qualification, <, my_type) +#define DECLARE_NEQ(prefix, qualification, my_type) \ + DECLARE_ONE_CMP(prefix, qualification, !=, my_type) + +#define GENERATE_ONE_CMP(PRE, QUAL, COMPARATOR, MY_TYPE, ...) \ + PRE bool QUAL operator COMPARATOR(const MY_TYPE & other) const { \ + __VA_OPT__(const MY_TYPE * me = this;) \ + auto fields1 = std::make_tuple( __VA_ARGS__ ); \ + __VA_OPT__(me = &other;) \ + auto fields2 = std::make_tuple( __VA_ARGS__ ); \ + return fields1 COMPARATOR fields2; \ + } +#define GENERATE_EQUAL(prefix, qualification, my_type, args...) \ + GENERATE_ONE_CMP(prefix, qualification, ==, my_type, args) +#define GENERATE_LEQ(prefix, qualification, my_type, args...) \ + GENERATE_ONE_CMP(prefix, qualification, <, my_type, args) +#define GENERATE_NEQ(prefix, qualification, my_type, args...) \ + GENERATE_ONE_CMP(prefix, qualification, !=, my_type, args) + /** * Declare comparison methods without defining them. */ -#define DECLARE_ONE_CMP(COMPARATOR, MY_TYPE) \ - bool operator COMPARATOR(const MY_TYPE & other) const; -#define DECLARE_EQUAL(my_type) \ - DECLARE_ONE_CMP(==, my_type) -#define DECLARE_LEQ(my_type) \ - DECLARE_ONE_CMP(<, my_type) -#define DECLARE_NEQ(my_type) \ - DECLARE_ONE_CMP(!=, my_type) #define DECLARE_CMP(my_type) \ - DECLARE_EQUAL(my_type) \ - DECLARE_LEQ(my_type) \ - DECLARE_NEQ(my_type) + DECLARE_EQUAL(,,my_type) \ + DECLARE_LEQ(,,my_type) \ + DECLARE_NEQ(,,my_type) + +/** + * @param prefix This is for something before each declaration like + * `template`. + * + * @param my_type the type are defining operators for. + */ +#define DECLARE_CMP_EXT(prefix, qualification, my_type) \ + DECLARE_EQUAL(prefix, qualification, my_type) \ + DECLARE_LEQ(prefix, qualification, my_type) \ + DECLARE_NEQ(prefix, qualification, my_type) /** * Awful hacky generation of the comparison operators by doing a lexicographic @@ -33,18 +60,18 @@ * } * ``` */ -#define GENERATE_ONE_CMP(COMPARATOR, MY_TYPE, ...) \ - bool operator COMPARATOR(const MY_TYPE& other) const { \ - __VA_OPT__(const MY_TYPE* me = this;) \ - auto fields1 = std::make_tuple( __VA_ARGS__ ); \ - __VA_OPT__(me = &other;) \ - auto fields2 = std::make_tuple( __VA_ARGS__ ); \ - return fields1 COMPARATOR fields2; \ - } -#define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args) -#define GENERATE_LEQ(args...) GENERATE_ONE_CMP(<, args) -#define GENERATE_NEQ(args...) GENERATE_ONE_CMP(!=, args) #define GENERATE_CMP(args...) \ - GENERATE_EQUAL(args) \ - GENERATE_LEQ(args) \ - GENERATE_NEQ(args) + GENERATE_EQUAL(,,args) \ + GENERATE_LEQ(,,args) \ + GENERATE_NEQ(,,args) + +/** + * @param prefix This is for something before each declaration like + * `template`. + * + * @param my_type the type are defining operators for. + */ +#define GENERATE_CMP_EXT(prefix, my_type, args...) \ + GENERATE_EQUAL(prefix, my_type ::, my_type, args) \ + GENERATE_LEQ(prefix, my_type ::, my_type, args) \ + GENERATE_NEQ(prefix, my_type ::, my_type, args) diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 66f319c3e..e2189fc66 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -406,8 +406,22 @@ static void main_nix_build(int argc, char * * argv) } } + std::function, const DerivedPathMap::ChildNode &)> accumDerivedPath; + + accumDerivedPath = [&](ref inputDrv, const DerivedPathMap::ChildNode & inputNode) { + if (!inputNode.value.empty()) + pathsToBuild.push_back(DerivedPath::Built { + .drvPath = inputDrv, + .outputs = OutputsSpec::Names { inputNode.value }, + }); + for (const auto & [outputName, childNode] : inputNode.childMap) + accumDerivedPath( + make_ref(SingleDerivedPath::Built { inputDrv, outputName }), + childNode); + }; + // Build or fetch all dependencies of the derivation. - for (const auto & [inputDrv0, inputOutputs] : drv.inputDrvs) { + for (const auto & [inputDrv0, inputNode] : drv.inputDrvs.map) { // To get around lambda capturing restrictions in the // standard. const auto & inputDrv = inputDrv0; @@ -416,10 +430,7 @@ static void main_nix_build(int argc, char * * argv) return !std::regex_search(store->printStorePath(inputDrv), std::regex(exclude)); })) { - pathsToBuild.push_back(DerivedPath::Built { - .drvPath = makeConstantStorePathRef(inputDrv), - .outputs = OutputsSpec::Names { inputOutputs }, - }); + accumDerivedPath(makeConstantStorePathRef(inputDrv), inputNode); pathsToCopy.insert(inputDrv); } } @@ -482,13 +493,21 @@ static void main_nix_build(int argc, char * * argv) if (env.count("__json")) { StorePathSet inputs; - for (auto & [depDrvPath, wantedDepOutputs] : drv.inputDrvs) { - auto outputs = evalStore->queryPartialDerivationOutputMap(depDrvPath); - for (auto & i : wantedDepOutputs) { + + std::function::ChildNode &)> accumInputClosure; + + accumInputClosure = [&](const StorePath & inputDrv, const DerivedPathMap::ChildNode & inputNode) { + auto outputs = evalStore->queryPartialDerivationOutputMap(inputDrv); + for (auto & i : inputNode.value) { auto o = outputs.at(i); store->computeFSClosure(*o, inputs); } - } + for (const auto & [outputName, childNode] : inputNode.childMap) + accumInputClosure(*outputs.at(outputName), childNode); + }; + + for (const auto & [inputDrv, inputNode] : drv.inputDrvs.map) + accumInputClosure(inputDrv, inputNode); ParsedDerivation parsedDrv(drvInfo.requireDrvPath(), drv); diff --git a/src/nix/app.cc b/src/nix/app.cc index 67c5ef344..34fac9935 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -20,15 +20,22 @@ StringPairs resolveRewrites( const std::vector & dependencies) { StringPairs res; - for (auto & dep : dependencies) - if (auto drvDep = std::get_if(&dep.path)) - if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) - for (auto & [ outputName, outputPath ] : drvDep->outputs) + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { + for (auto & dep : dependencies) { + if (auto drvDep = std::get_if(&dep.path)) { + for (auto & [ outputName, outputPath ] : drvDep->outputs) { res.emplace( - DownstreamPlaceholder::unknownCaOutput( - drvDep->drvPath->outPath(), outputName).render(), + DownstreamPlaceholder::fromSingleDerivedPathBuilt( + SingleDerivedPath::Built { + .drvPath = make_ref(drvDep->drvPath->discardOutputPath()), + .output = outputName, + }).render(), store.printStorePath(outputPath) ); + } + } + } + } return res; } diff --git a/tests/dyn-drv/dep-built-drv.sh b/tests/dyn-drv/dep-built-drv.sh index 8c4a45e3b..c3daf2c59 100644 --- a/tests/dyn-drv/dep-built-drv.sh +++ b/tests/dyn-drv/dep-built-drv.sh @@ -6,4 +6,6 @@ out1=$(nix-build ./text-hashed-output.nix -A hello --no-out-link) clearStore -expectStderr 1 nix-build ./text-hashed-output.nix -A wrapper --no-out-link | grepQuiet "Dependencies on the outputs of dynamic derivations are not yet supported" +out2=$(nix-build ./text-hashed-output.nix -A wrapper --no-out-link) + +diff -r $out1 $out2 From 80d7994f52ccefca2f2fbe4ee64741a6f49884ff Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 6 Sep 2023 22:57:37 -0400 Subject: [PATCH 070/139] Special-case error message to add extra information The Derivation parser and old ATerm unfortunately leaves few ways to get nice errors when an old version of Nix encounters a new version of the format. The most likely scenario for this to occur is with a new client making a derivation that the old daemon it is communicating with cannot understand. The extensions we just created for dynamic derivation deps will add a version field, solving the problem going forward, but there is still the issue of what to do about old versions of Nix up to now. The solution here is to carefully catch the bad error from the daemon that is likely to indicate this problem, and add some extra context to it. There is another "Ugly backwards compatibility hack" in `remote-store.cc` that also works by transforming an error. Co-authored-by: Robert Hensing --- src/libstore/remote-store.cc | 19 ++++++++++++++++++- tests/dyn-drv/local.mk | 3 ++- tests/dyn-drv/old-daemon-error-hack.nix | 20 ++++++++++++++++++++ tests/dyn-drv/old-daemon-error-hack.sh | 11 +++++++++++ 4 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 tests/dyn-drv/old-daemon-error-hack.nix create mode 100644 tests/dyn-drv/old-daemon-error-hack.sh diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 58f72beb9..a639346d1 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -172,7 +172,24 @@ void RemoteStore::ConnectionHandle::processStderr(Sink * sink, Source * source, auto ex = handle->processStderr(sink, source, flush); if (ex) { daemonException = true; - std::rethrow_exception(ex); + try { + std::rethrow_exception(ex); + } catch (const Error & e) { + // Nix versions before #4628 did not have an adequate behavior for reporting that the derivation format was upgraded. + // To avoid having to add compatibility logic in many places, we expect to catch almost all occurrences of the + // old incomprehensible error here, so that we can explain to users what's going on when their daemon is + // older than #4628 (2023). + if (experimentalFeatureSettings.isEnabled(Xp::DynamicDerivations) && + GET_PROTOCOL_MINOR(handle->daemonVersion) <= 35) + { + auto m = e.msg(); + if (m.find("parsing derivation") != std::string::npos && + m.find("expected string") != std::string::npos && + m.find("Derive([") != std::string::npos) + throw Error("%s, this might be because the daemon is too old to understand dependencies on dynamic derivations. Check to see if the raw dervation is in the form '%s'", std::move(m), "DrvWithVersion(..)"); + } + throw; + } } } diff --git a/tests/dyn-drv/local.mk b/tests/dyn-drv/local.mk index 0ce7cd37d..6b435499b 100644 --- a/tests/dyn-drv/local.mk +++ b/tests/dyn-drv/local.mk @@ -3,7 +3,8 @@ dyn-drv-tests := \ $(d)/recursive-mod-json.sh \ $(d)/build-built-drv.sh \ $(d)/eval-outputOf.sh \ - $(d)/dep-built-drv.sh + $(d)/dep-built-drv.sh \ + $(d)/old-daemon-error-hack.sh install-tests-groups += dyn-drv diff --git a/tests/dyn-drv/old-daemon-error-hack.nix b/tests/dyn-drv/old-daemon-error-hack.nix new file mode 100644 index 000000000..c9d4a62d4 --- /dev/null +++ b/tests/dyn-drv/old-daemon-error-hack.nix @@ -0,0 +1,20 @@ +with import ./config.nix; + +# A simple content-addressed derivation. +# The derivation can be arbitrarily modified by passing a different `seed`, +# but the output will always be the same +rec { + stub = mkDerivation { + name = "stub"; + buildCommand = '' + echo stub > $out + ''; + }; + wrapper = mkDerivation { + name = "has-dynamic-drv-dep"; + buildCommand = '' + exit 1 # we're not building this derivation + ${builtins.outputOf stub.outPath "out"} + ''; + }; +} diff --git a/tests/dyn-drv/old-daemon-error-hack.sh b/tests/dyn-drv/old-daemon-error-hack.sh new file mode 100644 index 000000000..43b049973 --- /dev/null +++ b/tests/dyn-drv/old-daemon-error-hack.sh @@ -0,0 +1,11 @@ +# Purposely bypassing our usual common for this subgroup +source ../common.sh + +# Need backend to support text-hashing too +isDaemonNewer "2.18.0pre20230906" && skipTest "Daemon is too new" + +enableFeatures "ca-derivations dynamic-derivations" + +restartDaemon + +expectStderr 1 nix-instantiate --read-write-mode ./old-daemon-error-hack.nix | grepQuiet "the daemon is too old to understand dependencies on dynamic derivations" From 5473e10249e02f95375b6126e232700d51bdf429 Mon Sep 17 00:00:00 2001 From: thenbe <33713262+thenbe@users.noreply.github.com> Date: Thu, 7 Sep 2023 22:25:32 +0000 Subject: [PATCH 071/139] fix: `nix shell` multiple commands example (#8950) The `-c` flag belongs to `sh` not `nix shell`. As it stands, the command errors with: ``` $ nix shell nixpkgs#gnumake --command sh --command "cd src && make" sh: --command: invalid option ``` https://github.com/NixOS/nix/pull/8276 was good for readability, but it missed this since that PR used a find/replace script. --- src/nix/shell.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/shell.md b/src/nix/shell.md index 1668104b1..f36919575 100644 --- a/src/nix/shell.md +++ b/src/nix/shell.md @@ -26,7 +26,7 @@ R""( * Run multiple commands in a shell environment: ```console - # nix shell nixpkgs#gnumake --command sh --command "cd src && make" + # nix shell nixpkgs#gnumake --command sh -c "cd src && make" ``` * Run GNU Hello in a chroot store: From 2cdc9c32e746a620458a1dd87655d2a6c3800f64 Mon Sep 17 00:00:00 2001 From: Emil Nikolov Date: Sat, 9 Sep 2023 08:54:39 +0200 Subject: [PATCH 072/139] docs: fixed the default priority of nix-env --install (#8945) --- doc/manual/src/command-ref/nix-env/install.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/command-ref/nix-env/install.md b/doc/manual/src/command-ref/nix-env/install.md index ad179cbc7..5dc04c385 100644 --- a/doc/manual/src/command-ref/nix-env/install.md +++ b/doc/manual/src/command-ref/nix-env/install.md @@ -30,7 +30,7 @@ a number of possible ways: derivation with the highest *priority* is used. A derivation can define a priority by declaring the `meta.priority` attribute. This attribute should be a number, with a higher value denoting a lower - priority. The default priority is `0`. + priority. The default priority is `5`. If there are multiple matching derivations with the same priority, then the derivation with the highest version will be installed. From 682dbcab9a7a81c210dc9c8aa3cfc58ccc1723aa Mon Sep 17 00:00:00 2001 From: maralorn Date: Sat, 9 Sep 2023 18:01:10 +0200 Subject: [PATCH 073/139] Print parent activity field in json log --- src/libutil/logging.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 54702e4ea..9d7a141b3 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -220,8 +220,8 @@ struct JSONLogger : Logger { json["level"] = lvl; json["type"] = type; json["text"] = s; + json["parent"] = parent; addFields(json, fields); - // FIXME: handle parent write(json); } From 07545add53ab39b4560d8e7cec1748423ae1a22e Mon Sep 17 00:00:00 2001 From: Matthew Kenigsberg Date: Sun, 10 Sep 2023 12:18:03 +0200 Subject: [PATCH 074/139] Drop dead code localPath is unused --- src/libexpr/flake/flake.cc | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 6a27ea2e8..90427e064 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -520,11 +520,6 @@ LockedFlake lockFlake( } } - auto localPath(parentPath); - // If this input is a path, recurse it down. - // This allows us to resolve path inputs relative to the current flake. - if ((*input.ref).input.getType() == "path") - localPath = absPath(*input.ref->input.getSourcePath(), parentPath); computeLocks( mustRefetch ? getFlake(state, oldLock->lockedRef, false, flakeCache, inputPath).inputs From 3720e811fa2883bca1a2698543146070e4d8d32c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 12 Sep 2023 13:21:55 +0200 Subject: [PATCH 075/139] libexpr: Add nrExprs to NIX_SHOW_STATS --- src/libexpr/eval.cc | 1 + src/libexpr/nixexpr.hh | 4 ++++ src/libexpr/parser.y | 1 + 3 files changed, 6 insertions(+) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 6d445fd96..afa864730 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2506,6 +2506,7 @@ void EvalState::printStats() {"elements", nrValuesInEnvs}, {"bytes", bEnvs}, }; + topObj["nrExprs"] = Expr::nrExprs; topObj["list"] = { {"elements", nrListElems}, {"bytes", bLists}, diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 5ca3d1fa6..b80c07f46 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -155,6 +155,10 @@ std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath) struct Expr { + static unsigned long nrExprs; + inline Expr() { + nrExprs++; + } virtual ~Expr() { }; virtual void show(const SymbolTable & symbols, std::ostream & str) const; virtual void bindVars(EvalState & es, const std::shared_ptr & env); diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 792f51fde..a3d3c7041 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -653,6 +653,7 @@ formal namespace nix { +unsigned long Expr::nrExprs = 0; Expr * EvalState::parse( char * text, From bf8deb4991286a44e1b6cc2142721a594276dd12 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 12 Sep 2023 13:45:45 +0200 Subject: [PATCH 076/139] Expr: remove redundant int and float fields --- src/libexpr/nixexpr.cc | 4 ++-- src/libexpr/nixexpr.hh | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 4566a1388..22be8e68c 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -76,12 +76,12 @@ void Expr::show(const SymbolTable & symbols, std::ostream & str) const void ExprInt::show(const SymbolTable & symbols, std::ostream & str) const { - str << n; + str << v.integer; } void ExprFloat::show(const SymbolTable & symbols, std::ostream & str) const { - str << nf; + str << v.fpoint; } void ExprString::show(const SymbolTable & symbols, std::ostream & str) const diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index b80c07f46..17b2d1136 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -175,18 +175,16 @@ struct Expr struct ExprInt : Expr { - NixInt n; Value v; - ExprInt(NixInt n) : n(n) { v.mkInt(n); }; + ExprInt(NixInt n) { v.mkInt(n); }; Value * maybeThunk(EvalState & state, Env & env) override; COMMON_METHODS }; struct ExprFloat : Expr { - NixFloat nf; Value v; - ExprFloat(NixFloat nf) : nf(nf) { v.mkFloat(nf); }; + ExprFloat(NixFloat nf) { v.mkFloat(nf); }; Value * maybeThunk(EvalState & state, Env & env) override; COMMON_METHODS }; From 21783cff1649f236cb31f27f788e3934802c42c9 Mon Sep 17 00:00:00 2001 From: Emil Nikolov Date: Tue, 12 Sep 2023 17:15:36 +0200 Subject: [PATCH 077/139] docs: make the nix develop --command example unambiguous (#8952) --- src/nix/develop.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/develop.md b/src/nix/develop.md index 1b5a8aeba..c49b39669 100644 --- a/src/nix/develop.md +++ b/src/nix/develop.md @@ -69,7 +69,7 @@ R""( * Run a series of script commands: ```console - # nix develop --command bash --command "mkdir build && cmake .. && make" + # nix develop --command bash -c "mkdir build && cmake .. && make" ``` # Description From 408055a9ddf855e32a59e07e564ad893b2cb2cdf Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Mon, 18 Sep 2023 20:08:48 +0200 Subject: [PATCH 078/139] add clarifying example to `nix-env` output selection there is a very confusing warning in the Nixpkgs manual that mischaracterises `nix-env` behavior, and this example shows what's really happening. note that it doesn't use `pkgs.runCommand` or other `pkgs.stdenv` facilities, as deep down those set `meta.outputsToInstall` to very particular defaults that do not generally apply to Nix. --- doc/manual/src/command-ref/nix-env/install.md | 60 ++++++++++++++++++- 1 file changed, 57 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/command-ref/nix-env/install.md b/doc/manual/src/command-ref/nix-env/install.md index 5dc04c385..95648e9e0 100644 --- a/doc/manual/src/command-ref/nix-env/install.md +++ b/doc/manual/src/command-ref/nix-env/install.md @@ -19,12 +19,15 @@ current generation of the active profile, to which a set of store paths described by *args* is added. The arguments *args* map to store paths in a number of possible ways: - - By default, *args* is a set of derivation names denoting derivations + + - By default, *args* is a set of [derivation] names denoting derivations in the active Nix expression. These are realised, and the resulting output paths are installed. Currently installed derivations with a name equal to the name of a derivation being added are removed unless the option `--preserve-installed` is specified. + [derivation]: @docroot@/language/derivations.md + If there are multiple derivations matching a name in *args* that have the same name (e.g., `gcc-3.3.6` and `gcc-4.1.1`), then the derivation with the highest *priority* is used. A derivation can @@ -66,8 +69,59 @@ a number of possible ways: - If *args* are store paths that are not store derivations, then these are [realised](@docroot@/command-ref/nix-store/realise.md) and installed. - - By default all outputs are installed for each derivation. That can - be reduced by setting `meta.outputsToInstall`. + - By default all outputs are installed for each derivation. + This can be overridden by adding a `meta.outputsToInstall` attribute on the derivation listing a subset of the output names. + + + + Example: + + The file `example.nix` defines a [derivation] with two outputs `foo` and `bar`, each containing a file. + + ```nix + # example.nix + let + pkgs = import {}; + command = '' + ${pkgs.coreutils}/bin/mkdir -p $foo $bar + echo foo > $foo/foo-file + echo bar > $bar/bar-file + ''; + in + derivation { + name = "example"; + builder = "${pkgs.bash}/bin/bash"; + args = [ "-c" command ]; + outputs = [ "foo" "bar" ]; + system = builtins.currentSystem; + } + ``` + + Installing from this Nix expression will make files from both outputs appear in the current profile. + + ```console + $ nix-env --install --file example.nix + installing 'example' + $ ls ~/.nix-profile + foo-file + bar-file + manifest.nix + ``` + + Adding `meta.outputsToInstall` to that derivation will make `nix-env` only install files from the specified outputs. + + ```nix + # example-outputs.nix + import ./example.nix // { meta.outputsToInstall = [ "bar" ]; } + ``` + + ```console + $ nix-env --install --file example-outputs.nix + installing 'example' + $ ls ~/.nix-profile + bar-file + manifest.nix + ``` # Flags From dd3bf4dbda02b83a185efc1f8f4bced06a6b07ab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 22:16:55 +0000 Subject: [PATCH 079/139] Bump docker/login-action from 2 to 3 Bumps [docker/login-action](https://github.com/docker/login-action) from 2 to 3. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/v2...v3) --- updated-dependencies: - dependency-name: docker/login-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 515548a4e..0c9c24dad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -127,7 +127,7 @@ jobs: - run: docker tag nix:$NIX_VERSION nixos/nix:$NIX_VERSION - run: docker tag nix:$NIX_VERSION nixos/nix:master - name: Login to Docker Hub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} From c8afa01bc25ba81883f8007ef83532ca550c731d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 19 Sep 2023 14:51:50 +0200 Subject: [PATCH 080/139] Try aws-sdk-cpp fix --- flake.lock | 12 ++++++------ flake.nix | 3 ++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/flake.lock b/flake.lock index 2bc503258..75b6ae6a7 100644 --- a/flake.lock +++ b/flake.lock @@ -34,16 +34,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1693494129, - "narHash": "sha256-YrHlSbniFmhcz0ORe8MMFttifKR4hTRzyX2OQUO9VxA=", - "owner": "NixOS", + "lastModified": 1695124524, + "narHash": "sha256-trXDytVCqf3KryQQQrHOZKUabu1/lB8/ndOAuZKQrOE=", + "owner": "edolstra", "repo": "nixpkgs", - "rev": "22a584b861ab31a2dca52121999079832f0e0f73", + "rev": "a3d30b525535e3158221abc1a957ce798ab159fe", "type": "github" }, "original": { - "owner": "NixOS", - "ref": "nixos-23.05-small", + "owner": "edolstra", + "ref": "fix-aws-sdk-cpp", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index 49aa9c4ce..249da2a04 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,8 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; + #inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; + inputs.nixpkgs.url = "github:edolstra/nixpkgs/fix-aws-sdk-cpp"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; From 126e2645f2a060197655f9a699ffa4dc3a464bdd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 19 Sep 2023 16:04:00 +0200 Subject: [PATCH 081/139] Disable rapidcheck tests in the coverage run https://hydra.nixos.org/build/233688539 --- flake.nix | 2 ++ src/libexpr/tests/derived-path.cc | 4 ++++ src/libexpr/tests/value/context.cc | 4 ++++ src/libstore/tests/derived-path.cc | 4 ++++ src/libstore/tests/outputs-spec.cc | 4 ++++ src/libstore/tests/path.cc | 4 ++++ 6 files changed, 22 insertions(+) diff --git a/flake.nix b/flake.nix index 249da2a04..dfb2dc1f8 100644 --- a/flake.nix +++ b/flake.nix @@ -588,6 +588,8 @@ lcovFilter = [ "*/boost/*" "*-tab.*" ]; hardeningDisable = ["fortify"]; + + NIX_CFLAGS_COMPILE = "-DCOVERAGE=1"; }; # API docs for Nix's unstable internal C++ interfaces. diff --git a/src/libexpr/tests/derived-path.cc b/src/libexpr/tests/derived-path.cc index d01735db8..d5fc6f201 100644 --- a/src/libexpr/tests/derived-path.cc +++ b/src/libexpr/tests/derived-path.cc @@ -18,6 +18,8 @@ TEST_F(DerivedPathExpressionTest, force_init) { } +#ifndef COVERAGE + RC_GTEST_FIXTURE_PROP( DerivedPathExpressionTest, prop_opaque_path_round_trip, @@ -61,4 +63,6 @@ RC_GTEST_FIXTURE_PROP( RC_ASSERT(SingleDerivedPath { b } == d); } +#endif + } /* namespace nix */ diff --git a/src/libexpr/tests/value/context.cc b/src/libexpr/tests/value/context.cc index 19407d071..92d4889ab 100644 --- a/src/libexpr/tests/value/context.cc +++ b/src/libexpr/tests/value/context.cc @@ -147,6 +147,8 @@ Gen Arbitrary::arbitrary() namespace nix { +#ifndef COVERAGE + RC_GTEST_PROP( NixStringContextElemTest, prop_round_rip, @@ -155,4 +157,6 @@ RC_GTEST_PROP( RC_ASSERT(o == NixStringContextElem::parse(o.to_string())); } +#endif + } diff --git a/src/libstore/tests/derived-path.cc b/src/libstore/tests/derived-path.cc index d6549f66f..3fa3c0801 100644 --- a/src/libstore/tests/derived-path.cc +++ b/src/libstore/tests/derived-path.cc @@ -130,6 +130,8 @@ TEST_F(DerivedPathTest, built_built_xp) { MissingExperimentalFeature); } +#ifndef COVERAGE + RC_GTEST_FIXTURE_PROP( DerivedPathTest, prop_legacy_round_rip, @@ -146,4 +148,6 @@ RC_GTEST_FIXTURE_PROP( RC_ASSERT(o == DerivedPath::parse(*store, o.to_string(*store))); } +#endif + } diff --git a/src/libstore/tests/outputs-spec.cc b/src/libstore/tests/outputs-spec.cc index bf8deaa9d..952945185 100644 --- a/src/libstore/tests/outputs-spec.cc +++ b/src/libstore/tests/outputs-spec.cc @@ -224,6 +224,8 @@ Gen Arbitrary::arbitrary() namespace nix { +#ifndef COVERAGE + RC_GTEST_PROP( OutputsSpec, prop_round_rip, @@ -232,4 +234,6 @@ RC_GTEST_PROP( RC_ASSERT(o == OutputsSpec::parse(o.to_string())); } +#endif + } diff --git a/src/libstore/tests/path.cc b/src/libstore/tests/path.cc index 430aa0099..efa35ef2b 100644 --- a/src/libstore/tests/path.cc +++ b/src/libstore/tests/path.cc @@ -134,6 +134,8 @@ Gen Arbitrary::arbitrary() namespace nix { +#ifndef COVERAGE + RC_GTEST_FIXTURE_PROP( StorePathTest, prop_regex_accept, @@ -150,4 +152,6 @@ RC_GTEST_FIXTURE_PROP( RC_ASSERT(p == store->parseStorePath(store->printStorePath(p))); } +#endif + } From c6953d1ff62fb6dc4fbd89c03e7949c552c19382 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 19 Sep 2023 17:03:21 +0200 Subject: [PATCH 082/139] Disable systemd-nspawn test This is broken because of a change in systemd in NixOS 23.05. It fails with Failed to mount proc (type proc) on /proc (MS_NOSUID|MS_NODEV|MS_NOEXEC ""): Operation not permitted --- tests/nixos/containers/containers.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/nixos/containers/containers.nix b/tests/nixos/containers/containers.nix index c8ee78a4a..e721be48f 100644 --- a/tests/nixos/containers/containers.nix +++ b/tests/nixos/containers/containers.nix @@ -56,8 +56,8 @@ host.fail("nix build -v --auto-allocate-uids --no-sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-6 --arg uidRange true") # Run systemd-nspawn in a Nix build. - host.succeed("nix build -v --auto-allocate-uids --sandbox -L --offline --impure --file ${./systemd-nspawn.nix} --argstr nixpkgs ${nixpkgs}") - host.succeed("[[ $(cat ./result/msg) = 'Hello World' ]]") + #host.succeed("nix build -v --auto-allocate-uids --sandbox -L --offline --impure --file ${./systemd-nspawn.nix} --argstr nixpkgs ${nixpkgs}") + #host.succeed("[[ $(cat ./result/msg) = 'Hello World' ]]") ''; } From b6b2a0aea99995d73f0faa6115fb33a137e57b23 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 19 Sep 2023 17:21:07 +0200 Subject: [PATCH 083/139] Use "touch -h" https://hydra.nixos.org/build/235888160 This is needed because Nixpkgs now contains dangling symlinks (pkgs/test/nixpkgs-check-by-name/tests/symlink-invalid/pkgs/by-name/fo/foo/foo.nix). --- tests/nixos/github-flakes.nix | 2 +- tests/nixos/sourcehut-flakes.nix | 2 +- tests/nixos/tarball-flakes.nix | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/nixos/github-flakes.nix b/tests/nixos/github-flakes.nix index e4d347691..6de702d17 100644 --- a/tests/nixos/github-flakes.nix +++ b/tests/nixos/github-flakes.nix @@ -82,7 +82,7 @@ let dir=NixOS-nixpkgs-${nixpkgs.shortRev} cp -prd ${nixpkgs} $dir # Set the correct timestamp in the tarball. - find $dir -print0 | xargs -0 touch -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} -- + find $dir -print0 | xargs -0 touch -h -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} -- tar cfz $out/archive/${nixpkgs.rev}.tar.gz $dir --hard-dereference ''; in diff --git a/tests/nixos/sourcehut-flakes.nix b/tests/nixos/sourcehut-flakes.nix index a76fed020..6e8d884a0 100644 --- a/tests/nixos/sourcehut-flakes.nix +++ b/tests/nixos/sourcehut-flakes.nix @@ -47,7 +47,7 @@ let cp -prd ${nixpkgs} $dir # Set the correct timestamp in the tarball. - find $dir -print0 | xargs -0 touch -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} -- + find $dir -print0 | xargs -0 touch -h -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} -- mkdir -p $out/archive tar cfz $out/archive/${nixpkgs.rev}.tar.gz $dir --hard-dereference diff --git a/tests/nixos/tarball-flakes.nix b/tests/nixos/tarball-flakes.nix index 1d43a5d04..e30d15739 100644 --- a/tests/nixos/tarball-flakes.nix +++ b/tests/nixos/tarball-flakes.nix @@ -11,7 +11,7 @@ let dir=nixpkgs-${nixpkgs.shortRev} cp -prd ${nixpkgs} $dir # Set the correct timestamp in the tarball. - find $dir -print0 | xargs -0 touch -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} -- + find $dir -print0 | xargs -0 touch -h -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} -- tar cfz $out/stable/${nixpkgs.rev}.tar.gz $dir --hard-dereference echo 'Redirect "/latest.tar.gz" "/stable/${nixpkgs.rev}.tar.gz"' > $out/.htaccess From 10ad052f7d14101525daabbd63a147eb0247c3a1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 20 Sep 2023 11:42:49 +0200 Subject: [PATCH 084/139] Release notes --- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/release-notes/rl-2.18.md | 28 ++++++++++++++++++++++++ doc/manual/src/release-notes/rl-next.md | 29 ------------------------- 3 files changed, 29 insertions(+), 29 deletions(-) create mode 100644 doc/manual/src/release-notes/rl-2.18.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 701a6ea69..c23186511 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -110,6 +110,7 @@ - [C++ style guide](contributing/cxx.md) - [Release Notes](release-notes/release-notes.md) - [Release X.Y (202?-??-??)](release-notes/rl-next.md) + - [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md) - [Release 2.17 (2023-07-24)](release-notes/rl-2.17.md) - [Release 2.16 (2023-05-31)](release-notes/rl-2.16.md) - [Release 2.15 (2023-04-11)](release-notes/rl-2.15.md) diff --git a/doc/manual/src/release-notes/rl-2.18.md b/doc/manual/src/release-notes/rl-2.18.md new file mode 100644 index 000000000..4bbc52b50 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.18.md @@ -0,0 +1,28 @@ +# Release 2.18 (2023-09-20) + +- Two new builtin functions, + [`builtins.parseFlakeRef`](@docroot@/language/builtins.md#builtins-parseFlakeRef) + and + [`builtins.flakeRefToString`](@docroot@/language/builtins.md#builtins-flakeRefToString), + have been added. + These functions are useful for converting between flake references encoded as attribute sets and URLs. + +- [`builtins.toJSON`](@docroot@/language/builtins.md#builtins-parseFlakeRef) now prints [--show-trace](@docroot@/command-ref/conf-file.html#conf-show-trace) items for the path in which it finds an evaluation error. + +- Error messages regarding malformed input to [`nix derivation add`](@docroot@/command-ref/new-cli/nix3-derivation-add.md) are now clearer and more detailed. + +- The `discard-references` feature has been stabilized. + This means that the + [unsafeDiscardReferences](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) + attribute is no longer guarded by an experimental flag and can be used + freely. + +- The JSON output for derived paths which are store paths is now a string, not an object with a single `path` field. + This only affects `nix-build --json` when "building" non-derivation things like fetched sources, which is a no-op. + +- A new builtin [`outputOf`](@docroot@/language/builtins.md#builtins-outputOf) has been added. + It is part of the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. + +- Flake follow paths at depths greater than 2 are now handled correctly, preventing "follows a non-existent input" errors. + +- [`nix-store --query`](@docroot@/command-ref/nix-store/query.md) gained a new type of query: `--valid-derivers`. It returns all `.drv` files in the local store that *can be* used to build the output passed in argument. This is in contrast to `--deriver`, which returns the single `.drv` file that *was actually* used to build the output passed in argument. In case the output was substituted from a binary cache, this `.drv` file may only exist on said binary cache and not locally. diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 02ad0a661..c869b5e2f 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,30 +1 @@ # Release X.Y (202?-??-??) - -- Two new builtin functions, - [`builtins.parseFlakeRef`](@docroot@/language/builtins.md#builtins-parseFlakeRef) - and - [`builtins.flakeRefToString`](@docroot@/language/builtins.md#builtins-flakeRefToString), - have been added. - These functions are useful for converting between flake references encoded as attribute sets and URLs. - -- [`builtins.toJSON`](@docroot@/language/builtins.md#builtins-parseFlakeRef) now prints [--show-trace](@docroot@/command-ref/conf-file.html#conf-show-trace) items for the path in which it finds an evaluation error. - -- Error messages regarding malformed input to [`derivation add`](@docroot@/command-ref/new-cli/nix3-derivation-add.md) are now clearer and more detailed. - -- The `discard-references` feature has been stabilized. - This means that the - [unsafeDiscardReferences](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) - attribute is no longer guarded by an experimental flag and can be used - freely. - -- The JSON output for derived paths with are store paths is now a string, not an object with a single `path` field. - This only affects `nix-build --json` when "building" non-derivation things like fetched sources, which is a no-op. - -- Introduce a new [`outputOf`](@docroot@/language/builtins.md#builtins-outputOf) builtin. - It is part of the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. - -- Flake follow paths at depths greater than 2 are now handled correctly, preventing "follows a non-existent input" errors. - -- [`nix-store --query`](@docroot@/command-ref/nix-store/query.md) gained a new type of query: `--valid-derivers`. It returns all `.drv` files in the local store that *can be* used to build the output passed in argument. -This is in contrast to `--deriver`, which returns the single `.drv` file that *was actually* used to build the output passed in argument. In case the output was substituted from a binary cache, -this `.drv` file may only exist on said binary cache and not locally. From e44d2a6bbef113b3d8f162f75bd5c94e0101075f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 30 Aug 2023 22:57:59 -0400 Subject: [PATCH 085/139] Add FreeBSD and NetBSD cross to Nix's flake --- flake.nix | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index dfb2dc1f8..871f0d7ba 100644 --- a/flake.nix +++ b/flake.nix @@ -25,8 +25,11 @@ linuxSystems = linux32BitSystems ++ linux64BitSystems; darwinSystems = [ "x86_64-darwin" "aarch64-darwin" ]; systems = linuxSystems ++ darwinSystems; - - crossSystems = [ "armv6l-linux" "armv7l-linux" ]; + + crossSystems = [ + "armv6l-linux" "armv7l-linux" + "x86_64-freebsd13" "x86_64-netbsd" + ]; stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" "libcxxStdenv" "ccacheStdenv" ]; @@ -94,7 +97,14 @@ nixpkgsFor = forAllSystems (system: let make-pkgs = crossSystem: stdenv: import nixpkgs { - inherit system crossSystem; + localSystem = { + inherit system; + }; + crossSystem = if crossSystem == null then null else { + system = crossSystem; + } // lib.optionalAttrs (crossSystem == "x86_64-freebsd13") { + useLLVM = true; + }; overlays = [ (overlayFor (p: p.${stdenv})) ]; From 28850ee90095e337bf47fbe03a0d937cb98784e2 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 30 Aug 2023 23:27:53 -0400 Subject: [PATCH 086/139] Make dev shells work for cross Need to get tools from right package set. Could build clang tools but I don't want to wait :D. --- flake.nix | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 871f0d7ba..24d9520ea 100644 --- a/flake.nix +++ b/flake.nix @@ -750,7 +750,11 @@ outputs = [ "out" "dev" "doc" ]; nativeBuildInputs = nativeBuildDeps - ++ (lib.optionals stdenv.cc.isClang [ pkgs.bear pkgs.clang-tools ]); + ++ lib.optional stdenv.cc.isClang pkgs.buildPackages.bear + ++ lib.optional + (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) + pkgs.buildPackages.clang-tools + ; buildInputs = buildDeps ++ propagatedDeps ++ awsDeps ++ checkDeps ++ internalApiDocsDeps; From 564392b57bc44c407303e4eaf6138d61a8ea50b0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 30 Aug 2023 23:29:17 -0400 Subject: [PATCH 087/139] Make libsodium an unconditional dependency The configure script will not tolerate it being missing. --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 24d9520ea..50a226347 100644 --- a/flake.nix +++ b/flake.nix @@ -190,9 +190,9 @@ libarchive boost lowdown-nix + libsodium ] ++ lib.optionals stdenv.isLinux [libseccomp] - ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid; checkDeps = [ From 0db251e4ad07338375fd59134fb467e9ebc4176a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 30 Aug 2023 23:43:48 -0400 Subject: [PATCH 088/139] Do not build docs in cross devShell Coppied from the main build; we really should deduplicate this more. --- flake.nix | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 50a226347..36937057d 100644 --- a/flake.nix +++ b/flake.nix @@ -743,6 +743,9 @@ devShells = let makeShell = pkgs: stdenv: + let + canRunInstalled = stdenv.buildPlatform.canExecute stdenv.hostPlatform; + in with commonDeps { inherit pkgs; }; stdenv.mkDerivation { name = "nix"; @@ -760,7 +763,8 @@ ++ awsDeps ++ checkDeps ++ internalApiDocsDeps; configureFlags = configureFlags - ++ testConfigureFlags ++ internalApiDocsConfigureFlags; + ++ testConfigureFlags ++ internalApiDocsConfigureFlags + ++ lib.optional (!canRunInstalled) "--disable-doc-gen"; enableParallelBuilding = true; From 7f76d7f038fbb5cb7982cf9aa951b9730566e275 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 30 Aug 2023 23:44:23 -0400 Subject: [PATCH 089/139] Rename an identifier of ours called `stdout` This is a reserved identifier on NetBSD --- it is replaced by a macro on that platform --- and so we cannot use it. --- src/libmain/shared.cc | 6 +++--- src/libmain/shared.hh | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 56f47a4ac..9c2ad039a 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -379,9 +379,9 @@ RunPager::RunPager() }); pid.setKillSignal(SIGINT); - stdout = fcntl(STDOUT_FILENO, F_DUPFD_CLOEXEC, 0); + std_out = fcntl(STDOUT_FILENO, F_DUPFD_CLOEXEC, 0); if (dup2(toPager.writeSide.get(), STDOUT_FILENO) == -1) - throw SysError("dupping stdout"); + throw SysError("dupping standard output"); } @@ -390,7 +390,7 @@ RunPager::~RunPager() try { if (pid != -1) { std::cout.flush(); - dup2(stdout, STDOUT_FILENO); + dup2(std_out, STDOUT_FILENO); pid.wait(); } } catch (...) { diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 7a9e83c6c..9415be78a 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -85,8 +85,9 @@ struct LegacyArgs : public MixCommonArgs void showManPage(const std::string & name); /** - * The constructor of this class starts a pager if stdout is a - * terminal and $PAGER is set. Stdout is redirected to the pager. + * The constructor of this class starts a pager if standard output is a + * terminal and $PAGER is set. Standard output is redirected to the + * pager. */ class RunPager { @@ -96,7 +97,7 @@ public: private: Pid pid; - int stdout; + int std_out; }; extern volatile ::sig_atomic_t blockInt; From c18911602eb4260d59acf8c17f1c3b4c7fcf7cee Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 31 Aug 2023 00:07:49 -0400 Subject: [PATCH 090/139] Fix `boehmgc-coroutine-sp-fallback.diff` for FreeBSD Our FreeBSD headers have `pthread_getattr_np`, but we get a link-time error that is missing. The good news is that there is another similar function which does exist, and the upstream project elsewhere does just the [fallback code] we need. As the fallback code indicates, the two functions are not identical however as the other one needs explicit initialization. NetBSD supports both in fact, and its [manpage] is therefore a good resource on what the differences are. [fallback code]: https://github.com/ivmai/bdwgc/blob/07a6d0ee8889bca5eaeadc13cabadc363725d216/os_dep.c#L1266-L1272 [manpage]: https://man.netbsd.org/pthread_attr_get_np.3 --- boehmgc-coroutine-sp-fallback.diff | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/boehmgc-coroutine-sp-fallback.diff b/boehmgc-coroutine-sp-fallback.diff index 5066d8278..2afbe9671 100644 --- a/boehmgc-coroutine-sp-fallback.diff +++ b/boehmgc-coroutine-sp-fallback.diff @@ -59,12 +59,18 @@ index b5d71e62..aed7b0bf 100644 GC_bool found_me = FALSE; size_t nthreads = 0; int i; -@@ -851,6 +853,31 @@ GC_INNER void GC_push_all_stacks(void) +@@ -851,6 +853,37 @@ GC_INNER void GC_push_all_stacks(void) hi = p->altstack + p->altstack_size; /* FIXME: Need to scan the normal stack too, but how ? */ /* FIXME: Assume stack grows down */ + } else { -+ if (pthread_getattr_np(p->id, &pattr)) { ++#ifdef HAVE_PTHREAD_ATTR_GET_NP ++ if (!pthread_attr_init(&pattr) ++ || !pthread_attr_get_np(p->id, &pattr)) ++#else /* HAVE_PTHREAD_GETATTR_NP */ ++ if (pthread_getattr_np(p->id, &pattr)) ++#endif ++ { + ABORT("GC_push_all_stacks: pthread_getattr_np failed!"); + } + if (pthread_attr_getstacksize(&pattr, &stack_limit)) { From b7acef1ceb756c72462a896d90c1b17ddc32a6df Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 20 Sep 2023 15:21:21 +0200 Subject: [PATCH 091/139] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index cf8690732..ef0f38abe 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.18.0 +2.19.0 From 883092e3f78d4efb1066a2e24e343b307035a04c Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Wed, 20 Sep 2023 09:09:01 -0700 Subject: [PATCH 092/139] Re-enable systemd-nspawn test It was disabled in c6953d1ff62fb6dc4fbd89c03e7949c552c19382 because a recent Nixpkgs bump brought in a new systemd which changed how systemd-nspawn worked. As far as I can tell, the issue was caused by this upstream systemd commit: https://github.com/systemd/systemd/commit/b71a0192c040f585397cfc6fc2ca025bf839733d Bind-mounting the host's `/sys` and `/proc` into the container's `/run/host/{sys,proc}` fixes the issue and allows the test to succeed. --- tests/nixos/containers/containers.nix | 4 ++-- tests/nixos/containers/systemd-nspawn.nix | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/nixos/containers/containers.nix b/tests/nixos/containers/containers.nix index e721be48f..c8ee78a4a 100644 --- a/tests/nixos/containers/containers.nix +++ b/tests/nixos/containers/containers.nix @@ -56,8 +56,8 @@ host.fail("nix build -v --auto-allocate-uids --no-sandbox -L --offline --impure --file ${./id-test.nix} --argstr name id-test-6 --arg uidRange true") # Run systemd-nspawn in a Nix build. - #host.succeed("nix build -v --auto-allocate-uids --sandbox -L --offline --impure --file ${./systemd-nspawn.nix} --argstr nixpkgs ${nixpkgs}") - #host.succeed("[[ $(cat ./result/msg) = 'Hello World' ]]") + host.succeed("nix build -v --auto-allocate-uids --sandbox -L --offline --impure --file ${./systemd-nspawn.nix} --argstr nixpkgs ${nixpkgs}") + host.succeed("[[ $(cat ./result/msg) = 'Hello World' ]]") ''; } diff --git a/tests/nixos/containers/systemd-nspawn.nix b/tests/nixos/containers/systemd-nspawn.nix index f54f32f2a..1dad4ebd7 100644 --- a/tests/nixos/containers/systemd-nspawn.nix +++ b/tests/nixos/containers/systemd-nspawn.nix @@ -73,6 +73,8 @@ runCommand "test" --resolv-conf=off \ --bind-ro=/nix/store \ --bind=$out \ + --bind=/proc:/run/host/proc \ + --bind=/sys:/run/host/sys \ --private-network \ $toplevel/init '' From e0e5943db28271cd6f1dcf56f3a0b12d0bc37b6c Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 21 Sep 2023 02:51:39 +0200 Subject: [PATCH 093/139] remove IRC from links in README the community has moved away from IRC a long time ago --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 85b0902b1..46a4218d9 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,6 @@ to set up a development environment and build Nix from source. - [Nix jobsets on hydra.nixos.org](https://hydra.nixos.org/project/nix) - [NixOS Discourse](https://discourse.nixos.org/) - [Matrix - #nix:nixos.org](https://matrix.to/#/#nix:nixos.org) -- [IRC - #nixos on libera.chat](irc://irc.libera.chat/#nixos) ## License From 747b2baf21f5bc3924945b02d10cd37b17e2e1bb Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 21 Sep 2023 02:51:54 +0200 Subject: [PATCH 094/139] fix rendering error for consecutive spaces --- doc/manual/src/language/index.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md index 29950a52d..a26e43a05 100644 --- a/doc/manual/src/language/index.md +++ b/doc/manual/src/language/index.md @@ -83,7 +83,8 @@ This is an incomplete overview of language features, by example. - A multi-line string. Strips common prefixed whitespace. Evaluates to `"multi\n line\n string"`. + + A multi-line string. Strips common prefixed whitespace. Evaluates to `"multi\n line\n  string"`. From 02649d247b7336697f059186ea8e9f3076dcda0a Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 21 Sep 2023 02:52:28 +0200 Subject: [PATCH 095/139] move test coverage section to testing page --- doc/manual/src/contributing/hacking.md | 14 -------------- doc/manual/src/contributing/testing.md | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 4b0a3a3e5..22f629c6f 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -271,17 +271,3 @@ or inside a `nix develop` shell by running: # make internal-api-html # xdg-open ./outputs/doc/share/doc/nix/internal-api/html/index.html ``` - -## Coverage analysis - -A coverage analysis report is [available -online](https://hydra.nixos.org/job/nix/master/coverage/latest/download-by-type/report/coverage). You -can build it yourself: - -``` -# nix build .#hydraJobs.coverage -# xdg-open ./result/coverage/index.html -``` - -Metrics about the change in line/function coverage over time are also -[available](https://hydra.nixos.org/job/nix/master/coverage#tabs-charts). diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index cd94d5cfb..881118505 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -1,5 +1,19 @@ # Running tests +## Coverage analysis + +A [coverage analysis report] is available online +You can build it yourself: + +[coverage analysis report]: https://hydra.nixos.org/job/nix/master/coverage/latest/download-by-type/report/coverage + +``` +# nix build .#hydraJobs.coverage +# xdg-open ./result/coverage/index.html +``` + +[Extensive records of build metrics](https://hydra.nixos.org/job/nix/master/coverage#tabs-charts), such as test coverage over time, are also available online. + ## Unit-tests The unit-tests for each Nix library (`libexpr`, `libstore`, etc..) are defined From 4685260a77266382b5c2c2b68ff1e6e0ddc8a908 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 21 Sep 2023 02:52:58 +0200 Subject: [PATCH 096/139] fix links to configuration settings --- src/libutil/experimental-features.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 782331283..211374bf6 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -171,7 +171,7 @@ constexpr std::array xpFeatureDetails = {{ .name = "auto-allocate-uids", .description = R"( Allows Nix to automatically pick UIDs for builds, rather than creating - `nixbld*` user accounts. See the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting for details. + `nixbld*` user accounts. See the [`auto-allocate-uids`](@docroot@/command-ref/conf-file.md#conf-auto-allocate-uids) setting for details. )", }, { @@ -179,7 +179,7 @@ constexpr std::array xpFeatureDetails = {{ .name = "cgroups", .description = R"( Allows Nix to execute builds inside cgroups. See - the [`use-cgroups`](#conf-use-cgroups) setting for details. + the [`use-cgroups`](@docroot@/command-ref/conf-file.md#conf-use-cgroups) setting for details. )", }, { From 984bd4cb0e2973ad756b6b1692919980f14f5df2 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 21 Sep 2023 02:53:36 +0200 Subject: [PATCH 097/139] README: link to CONTRIBUTING --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 85b0902b1..3b31b4d93 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,10 @@ Information on additional installation methods is available on the [Nix download See our [Hacking guide](https://nixos.org/manual/nix/unstable/contributing/hacking.html) in our manual for instruction on how to to set up a development environment and build Nix from source. +## Contributing + +Check the [contributing guide](./CONTRIBUTING.md) if you want to get involved with developing Nix. + ## Additional Resources - [Nix manual](https://nixos.org/nix/manual) From cf6ba7256f849ec1fe209b18c311e61ecd7719d3 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 21 Sep 2023 02:57:00 +0200 Subject: [PATCH 098/139] delete unused files --- doc/manual/src/command-ref/opt-common-syn.md | 57 -------------------- doc/manual/src/command-ref/opt-inst-syn.md | 15 ------ 2 files changed, 72 deletions(-) delete mode 100644 doc/manual/src/command-ref/opt-common-syn.md delete mode 100644 doc/manual/src/command-ref/opt-inst-syn.md diff --git a/doc/manual/src/command-ref/opt-common-syn.md b/doc/manual/src/command-ref/opt-common-syn.md deleted file mode 100644 index b66d318c2..000000000 --- a/doc/manual/src/command-ref/opt-common-syn.md +++ /dev/null @@ -1,57 +0,0 @@ -\--help - -\--version - -\--verbose - -\-v - -\--quiet - -\--log-format - -format - -\--no-build-output - -\-Q - -\--max-jobs - -\-j - -number - -\--cores - -number - -\--max-silent-time - -number - -\--timeout - -number - -\--keep-going - -\-k - -\--keep-failed - -\-K - -\--fallback - -\--readonly-mode - -\-I - -path - -\--option - -name - -value diff --git a/doc/manual/src/command-ref/opt-inst-syn.md b/doc/manual/src/command-ref/opt-inst-syn.md deleted file mode 100644 index 1703c40e3..000000000 --- a/doc/manual/src/command-ref/opt-inst-syn.md +++ /dev/null @@ -1,15 +0,0 @@ -\--prebuilt-only - -\-b - -\--attr - -\-A - -\--from-expression - -\-E - -\--from-profile - -path From 1a412a8d782b11b3af08be5b7ee3b4000b72cb32 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 21 Sep 2023 11:38:13 +0200 Subject: [PATCH 099/139] fix typo in docstring --- src/libutil/config.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/config.hh b/src/libutil/config.hh index cc8532587..96c2cd75d 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -36,8 +36,8 @@ namespace nix { * * std::map settings; * config.getSettings(settings); - * config["system"].description == "the current system" - * config["system"].value == "x86_64-linux" + * settings["system"].description == "the current system" + * settings["system"].value == "x86_64-linux" * * * The above retrieves all currently known settings from the `Config` object From f264d9ff0862277523a207c7afaed4894a40dc11 Mon Sep 17 00:00:00 2001 From: Gerg-L Date: Thu, 21 Sep 2023 21:00:35 -0400 Subject: [PATCH 100/139] flake: complete update to 23.05 --- flake.lock | 12 ++++++------ flake.nix | 3 +-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/flake.lock b/flake.lock index 75b6ae6a7..56df9c3fb 100644 --- a/flake.lock +++ b/flake.lock @@ -34,16 +34,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1695124524, - "narHash": "sha256-trXDytVCqf3KryQQQrHOZKUabu1/lB8/ndOAuZKQrOE=", - "owner": "edolstra", + "lastModified": 1695283060, + "narHash": "sha256-CJz71xhCLlRkdFUSQEL0pIAAfcnWFXMzd9vXhPrnrEg=", + "owner": "NixOS", "repo": "nixpkgs", - "rev": "a3d30b525535e3158221abc1a957ce798ab159fe", + "rev": "31ed632c692e6a36cfc18083b88ece892f863ed4", "type": "github" }, "original": { - "owner": "edolstra", - "ref": "fix-aws-sdk-cpp", + "owner": "NixOS", + "ref": "nixos-23.05-small", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index dfb2dc1f8..a7680a759 100644 --- a/flake.nix +++ b/flake.nix @@ -1,8 +1,7 @@ { description = "The purely functional package manager"; - #inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; - inputs.nixpkgs.url = "github:edolstra/nixpkgs/fix-aws-sdk-cpp"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; From d8cebae939932f2a5b458b5d92c82cd1a025dc0a Mon Sep 17 00:00:00 2001 From: Rasmus Rendal Date: Sat, 4 Jun 2022 12:07:08 +0200 Subject: [PATCH 101/139] Add a test for flake paths with spaces in them --- tests/flakes/common.sh | 10 +- tests/flakes/flakes.sh | 302 +++++++++++++++++++++-------------------- 2 files changed, 157 insertions(+), 155 deletions(-) diff --git a/tests/flakes/common.sh b/tests/flakes/common.sh index 427abcdde..8aed296e6 100644 --- a/tests/flakes/common.sh +++ b/tests/flakes/common.sh @@ -61,10 +61,10 @@ createGitRepo() { local repo="$1" local extraArgs="${2-}" - rm -rf $repo $repo.tmp - mkdir -p $repo + rm -rf "$repo" "$repo".tmp + mkdir -p "$repo" - git -C $repo init $extraArgs - git -C $repo config user.email "foobar@example.com" - git -C $repo config user.name "Foobar" + git -C "$repo" init $extraArgs + git -C "$repo" config user.email "foobar@example.com" + git -C "$repo" config user.name "Foobar" } diff --git a/tests/flakes/flakes.sh b/tests/flakes/flakes.sh index 128f759ea..70de28628 100644 --- a/tests/flakes/flakes.sh +++ b/tests/flakes/flakes.sh @@ -6,27 +6,29 @@ clearStore rm -rf $TEST_HOME/.cache $TEST_HOME/.config flake1Dir=$TEST_ROOT/flake1 -flake2Dir=$TEST_ROOT/flake2 -flake3Dir=$TEST_ROOT/flake3 +flake2Dir=$TEST_ROOT/flake\ 2 +percentEncodedFlake2Dir=$TEST_ROOT/flake%202 +flake3Dir=$TEST_ROOT/flake%20 +percentEncodedFlake3Dir=$TEST_ROOT/flake%2520 flake5Dir=$TEST_ROOT/flake5 flake7Dir=$TEST_ROOT/flake7 nonFlakeDir=$TEST_ROOT/nonFlake badFlakeDir=$TEST_ROOT/badFlake flakeGitBare=$TEST_ROOT/flakeGitBare -for repo in $flake1Dir $flake2Dir $flake3Dir $flake7Dir $nonFlakeDir; do +for repo in "$flake1Dir" "$flake2Dir" "$flake3Dir" "$flake7Dir" "$nonFlakeDir"; do # Give one repo a non-main initial branch. extraArgs= - if [[ $repo == $flake2Dir ]]; then + if [[ "$repo" == "$flake2Dir" ]]; then extraArgs="--initial-branch=main" fi createGitRepo "$repo" "$extraArgs" done -createSimpleGitFlake $flake1Dir +createSimpleGitFlake "$flake1Dir" -cat > $flake2Dir/flake.nix < "$flake2Dir/flake.nix" < $flake2Dir/flake.nix < $flake3Dir/flake.nix < "$flake3Dir/flake.nix" < $flake3Dir/flake.nix < $flake3Dir/default.nix < "$flake3Dir/default.nix" < $nonFlakeDir/README.md < "$nonFlakeDir/README.md" < $flake1Dir/foo -git -C $flake1Dir add $flake1Dir/foo +echo foo > "$flake1Dir/foo" +git -C "$flake1Dir" add $flake1Dir/foo [[ $(nix flake metadata flake1 --json --refresh | jq -r .dirtyRevision) == "$hash1-dirty" ]] -echo -n '# foo' >> $flake1Dir/flake.nix -flake1OriginalCommit=$(git -C $flake1Dir rev-parse HEAD) -git -C $flake1Dir commit -a -m 'Foo' -flake1NewCommit=$(git -C $flake1Dir rev-parse HEAD) +echo -n '# foo' >> "$flake1Dir/flake.nix" +flake1OriginalCommit=$(git -C "$flake1Dir" rev-parse HEAD) +git -C "$flake1Dir" commit -a -m 'Foo' +flake1NewCommit=$(git -C "$flake1Dir" rev-parse HEAD) hash2=$(nix flake metadata flake1 --json --refresh | jq -r .revision) [[ $(nix flake metadata flake1 --json --refresh | jq -r .dirtyRevision) == "null" ]] [[ $hash1 != $hash2 ]] # Test 'nix build' on a flake. -nix build -o $TEST_ROOT/result flake1#foo -[[ -e $TEST_ROOT/result/hello ]] +nix build -o "$TEST_ROOT/result" flake1#foo +[[ -e "$TEST_ROOT/result/hello" ]] # Test packages.default. -nix build -o $TEST_ROOT/result flake1 -[[ -e $TEST_ROOT/result/hello ]] +nix build -o "$TEST_ROOT/result" flake1 +[[ -e "$TEST_ROOT/result/hello" ]] -nix build -o $TEST_ROOT/result $flake1Dir -nix build -o $TEST_ROOT/result git+file://$flake1Dir +nix build -o "$TEST_ROOT/result" "$flake1Dir" +nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir" # Check that store symlinks inside a flake are not interpreted as flakes. -nix build -o $flake1Dir/result git+file://$flake1Dir -nix path-info $flake1Dir/result +nix build -o "$flake1Dir/result" "git+file://$flake1Dir" +nix path-info "$flake1Dir/result" # 'getFlake' on an unlocked flakeref should fail in pure mode, but # succeed in impure mode. -(! nix build -o $TEST_ROOT/result --expr "(builtins.getFlake \"$flake1Dir\").packages.$system.default") -nix build -o $TEST_ROOT/result --expr "(builtins.getFlake \"$flake1Dir\").packages.$system.default" --impure +(! nix build -o "$TEST_ROOT/result" --expr "(builtins.getFlake \"$flake1Dir\").packages.$system.default") +nix build -o "$TEST_ROOT/result" --expr "(builtins.getFlake \"$flake1Dir\").packages.$system.default" --impure # 'getFlake' on a locked flakeref should succeed even in pure mode. -nix build -o $TEST_ROOT/result --expr "(builtins.getFlake \"git+file://$flake1Dir?rev=$hash2\").packages.$system.default" +nix build -o "$TEST_ROOT/result" --expr "(builtins.getFlake \"git+file://$flake1Dir?rev=$hash2\").packages.$system.default" # Building a flake with an unlocked dependency should fail in pure mode. -(! nix build -o $TEST_ROOT/result flake2#bar --no-registries) -(! nix build -o $TEST_ROOT/result flake2#bar --no-use-registries) +(! nix build -o "$TEST_ROOT/result" flake2#bar --no-registries) +(! nix build -o "$TEST_ROOT/result" flake2#bar --no-use-registries) (! nix eval --expr "builtins.getFlake \"$flake2Dir\"") # But should succeed in impure mode. -(! nix build -o $TEST_ROOT/result flake2#bar --impure) -nix build -o $TEST_ROOT/result flake2#bar --impure --no-write-lock-file +(! nix build -o "$TEST_ROOT/result" flake2#bar --impure) +nix build -o "$TEST_ROOT/result" flake2#bar --impure --no-write-lock-file nix eval --expr "builtins.getFlake \"$flake2Dir\"" --impure # Building a local flake with an unlocked dependency should fail with --no-update-lock-file. -expect 1 nix build -o $TEST_ROOT/result $flake2Dir#bar --no-update-lock-file 2>&1 | grep 'requires lock file changes' +expect 1 nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --no-update-lock-file 2>&1 | grep 'requires lock file changes' # But it should succeed without that flag. -nix build -o $TEST_ROOT/result $flake2Dir#bar --no-write-lock-file -expect 1 nix build -o $TEST_ROOT/result $flake2Dir#bar --no-update-lock-file 2>&1 | grep 'requires lock file changes' -nix build -o $TEST_ROOT/result $flake2Dir#bar --commit-lock-file -[[ -e $flake2Dir/flake.lock ]] -[[ -z $(git -C $flake2Dir diff main || echo failed) ]] +nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --no-write-lock-file +expect 1 nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --no-update-lock-file 2>&1 | grep 'requires lock file changes' +nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --commit-lock-file +[[ -e "$flake2Dir/flake.lock" ]] +[[ -z $(git -C "$flake2Dir" diff main || echo failed) ]] # Rerunning the build should not change the lockfile. -nix build -o $TEST_ROOT/result $flake2Dir#bar -[[ -z $(git -C $flake2Dir diff main || echo failed) ]] +nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" +[[ -z $(git -C "$flake2Dir" diff main || echo failed) ]] # Building with a lockfile should not require a fetch of the registry. -nix build -o $TEST_ROOT/result --flake-registry file:///no-registry.json $flake2Dir#bar --refresh -nix build -o $TEST_ROOT/result --no-registries $flake2Dir#bar --refresh -nix build -o $TEST_ROOT/result --no-use-registries $flake2Dir#bar --refresh +nix build -o "$TEST_ROOT/result" --flake-registry file:///no-registry.json "$flake2Dir#bar" --refresh +nix build -o "$TEST_ROOT/result" --no-registries "$flake2Dir#bar" --refresh +nix build -o "$TEST_ROOT/result" --no-use-registries "$flake2Dir#bar" --refresh # Updating the flake should not change the lockfile. -nix flake lock $flake2Dir -[[ -z $(git -C $flake2Dir diff main || echo failed) ]] +nix flake lock "$flake2Dir" +[[ -z $(git -C "$flake2Dir" diff main || echo failed) ]] # Now we should be able to build the flake in pure mode. -nix build -o $TEST_ROOT/result flake2#bar +nix build -o "$TEST_ROOT/result" flake2#bar # Or without a registry. -nix build -o $TEST_ROOT/result --no-registries git+file://$flake2Dir#bar --refresh -nix build -o $TEST_ROOT/result --no-use-registries git+file://$flake2Dir#bar --refresh +nix build -o "$TEST_ROOT/result" --no-registries "git+file://$percentEncodedFlake2Dir#bar" --refresh +nix build -o "$TEST_ROOT/result" --no-use-registries "git+file://$percentEncodedFlake2Dir#bar" --refresh # Test whether indirect dependencies work. -nix build -o $TEST_ROOT/result $flake3Dir#xyzzy -git -C $flake3Dir add flake.lock +nix build -o "$TEST_ROOT/result" "$flake3Dir#xyzzy" +git -C "$flake3Dir" add flake.lock # Add dependency to flake3. -rm $flake3Dir/flake.nix +rm "$flake3Dir/flake.nix" -cat > $flake3Dir/flake.nix < "$flake3Dir/flake.nix" < $flake3Dir/flake.nix < $flake3Dir/flake.nix < "$flake3Dir/flake.nix" < $flake3Dir/flake.nix < $flake3Dir/flake.nix < "$flake3Dir/flake.nix" < $flake3Dir/flake.nix < $flake3Dir/flake.nix < "$flake3Dir/flake.nix" < $flake3Dir/flake.nix < $flake3Dir/flake.nix < "$flake3Dir/flake.nix" < $flake3Dir/flake.nix < $flake3Dir/flake.nix < "$flake3Dir/flake.nix" < $flake3Dir/flake.nix < $flake3Dir/flake.nix < "$flake3Dir/flake.nix" < $flake3Dir/flake.nix < $flake3Dir/flake.nix < "$flake3Dir/flake.nix" < $flake3Dir/flake.nix <' -A x) = 123 ]] [[ $(NIX_PATH=flake3=flake:flake3 nix-instantiate --eval '' -A x) = 123 ]] # Test alternate lockfile paths. -nix flake lock $flake2Dir --output-lock-file $TEST_ROOT/flake2.lock -cmp $flake2Dir/flake.lock $TEST_ROOT/flake2.lock >/dev/null # lockfiles should be identical, since we're referencing flake2's original one +nix flake lock "$flake2Dir" --output-lock-file $TEST_ROOT/flake2.lock +cmp "$flake2Dir/flake.lock" $TEST_ROOT/flake2.lock >/dev/null # lockfiles should be identical, since we're referencing flake2's original one -nix flake lock $flake2Dir --output-lock-file $TEST_ROOT/flake2-overridden.lock --override-input flake1 git+file://$flake1Dir?rev=$flake1OriginalCommit -expectStderr 1 cmp $flake2Dir/flake.lock $TEST_ROOT/flake2-overridden.lock -nix flake metadata $flake2Dir --reference-lock-file $TEST_ROOT/flake2-overridden.lock | grepQuiet $flake1OriginalCommit +nix flake lock "$flake2Dir" --output-lock-file $TEST_ROOT/flake2-overridden.lock --override-input flake1 git+file://$flake1Dir?rev=$flake1OriginalCommit +expectStderr 1 cmp "$flake2Dir/flake.lock" $TEST_ROOT/flake2-overridden.lock +nix flake metadata "$flake2Dir" --reference-lock-file $TEST_ROOT/flake2-overridden.lock | grepQuiet $flake1OriginalCommit # reference-lock-file can only be used if allow-dirty is set. -expectStderr 1 nix flake metadata $flake2Dir --no-allow-dirty --reference-lock-file $TEST_ROOT/flake2-overridden.lock +expectStderr 1 nix flake metadata "$flake2Dir" --no-allow-dirty --reference-lock-file $TEST_ROOT/flake2-overridden.lock From 50e61f579cfc8cf64031845f3e73f82593ad2ff3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Wed, 31 May 2023 10:36:43 +0200 Subject: [PATCH 102/139] Allow special characters in flake paths Support using nix flakes in paths with spaces or abitrary unicode characters. This introduces the convention that the path part of the URL should be percent-encoded when dealing with `path:` urls and not when using filepaths (following the convention of firefox). Co-authored-by: Rendal --- src/libexpr/flake/flakeref.cc | 85 +++++++++++++++++++---------------- src/libutil/tests/url.cc | 9 ++++ src/libutil/url.cc | 2 +- 3 files changed, 57 insertions(+), 39 deletions(-) diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index e1bce90bc..2155c5f12 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -77,14 +77,6 @@ std::pair parseFlakeRefWithFragment( { using namespace fetchers; - static std::string fnRegex = "[0-9a-zA-Z-._~!$&'\"()*+,;=]+"; - - static std::regex pathUrlRegex( - "(/?" + fnRegex + "(?:/" + fnRegex + ")*/?)" - + "(?:\\?(" + queryRegex + "))?" - + "(?:#(" + queryRegex + "))?", - std::regex::ECMAScript); - static std::regex flakeRegex( "((" + flakeIdRegexS + ")(?:/(?:" + refAndOrRevRegex + "))?)" + "(?:#(" + queryRegex + "))?", @@ -92,26 +84,23 @@ std::pair parseFlakeRefWithFragment( std::smatch match; - /* Check if 'url' is a flake ID. This is an abbreviated syntax for - 'flake:?ref=&rev='. */ - - if (std::regex_match(url, match, flakeRegex)) { - auto parsedURL = ParsedURL{ - .url = url, - .base = "flake:" + match.str(1), - .scheme = "flake", - .authority = "", - .path = match[1], - }; - - return std::make_pair( - FlakeRef(Input::fromURL(parsedURL, isFlake), ""), - percentDecode(match.str(6))); - } - - else if (std::regex_match(url, match, pathUrlRegex)) { - std::string path = match[1]; - std::string fragment = percentDecode(match.str(3)); + auto parsePathFlakeRef = [&]() { + std::string path = url; + std::string fragment = ""; + std::map query = {}; + auto pathEnd = url.find_first_of("#?"); + auto fragmentStart = pathEnd; + if (pathEnd != std::string::npos && url[pathEnd] == '?') + fragmentStart = url.find("#"); + if (pathEnd != std::string::npos) { + path = url.substr(0, pathEnd); + } + if (fragmentStart != std::string::npos) { + fragment = percentDecode(url.substr(fragmentStart+1)); + } + if (fragmentStart != std::string::npos && pathEnd != std::string::npos) { + query = decodeQuery(url.substr(pathEnd+1, fragmentStart)); + } if (baseDir) { /* Check if 'url' is a path (either absolute or relative @@ -163,7 +152,7 @@ std::pair parseFlakeRefWithFragment( .scheme = "git+file", .authority = "", .path = flakeRoot, - .query = decodeQuery(match[2]), + .query = query, }; if (subdir != "") { @@ -188,7 +177,6 @@ std::pair parseFlakeRefWithFragment( } else { if (!hasPrefix(path, "/")) throw BadURL("flake reference '%s' is not an absolute path", url); - auto query = decodeQuery(match[2]); path = canonPath(path + "/" + getOr(query, "dir", "")); } @@ -197,19 +185,40 @@ std::pair parseFlakeRefWithFragment( attrs.insert_or_assign("path", path); return std::make_pair(FlakeRef(Input::fromAttrs(std::move(attrs)), ""), fragment); + }; + + /* Check if 'url' is a flake ID. This is an abbreviated syntax for + 'flake:?ref=&rev='. */ + + if (std::regex_match(url, match, flakeRegex)) { + auto parsedURL = ParsedURL{ + .url = url, + .base = "flake:" + match.str(1), + .scheme = "flake", + .authority = "", + .path = match[1], + }; + + return std::make_pair( + FlakeRef(Input::fromURL(parsedURL), ""), + percentDecode(match.str(6))); } else { - auto parsedURL = parseURL(url); - std::string fragment; - std::swap(fragment, parsedURL.fragment); + try { + auto parsedURL = parseURL(url); + std::string fragment; + std::swap(fragment, parsedURL.fragment); - auto input = Input::fromURL(parsedURL, isFlake); - input.parent = baseDir; + auto input = Input::fromURL(parsedURL, isFlake); + input.parent = baseDir; - return std::make_pair( - FlakeRef(std::move(input), getOr(parsedURL.query, "dir", "")), - fragment); + return std::make_pair( + FlakeRef(std::move(input), getOr(parsedURL.query, "dir", "")), + fragment); + } catch (BadURL &) { + return parsePathFlakeRef(); + } } } diff --git a/src/libutil/tests/url.cc b/src/libutil/tests/url.cc index a908631e6..a678dad20 100644 --- a/src/libutil/tests/url.cc +++ b/src/libutil/tests/url.cc @@ -335,4 +335,13 @@ namespace nix { ASSERT_EQ(d, s); } + TEST(percentEncode, yen) { + // https://en.wikipedia.org/wiki/Percent-encoding#Character_data + std::string s = reinterpret_cast(u8"円"); + std::string e = "%E5%86%86"; + + ASSERT_EQ(percentEncode(s), e); + ASSERT_EQ(percentDecode(e), s); + } + } diff --git a/src/libutil/url.cc b/src/libutil/url.cc index a8f7d39fd..e700c8eaf 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -103,7 +103,7 @@ std::string percentEncode(std::string_view s, std::string_view keep) || keep.find(c) != std::string::npos) res += c; else - res += fmt("%%%02X", (unsigned int) c); + res += fmt("%%%02X", c & 0xFF); return res; } From e8113747e1abf7fa6225f7fcd31ca9e71e03cfbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= Date: Thu, 20 Apr 2023 06:51:47 +0200 Subject: [PATCH 103/139] Split the parseFlakeRefWithFragment function Was starting to be very complex and hard to follow. Now the different cases should be easier to understand. --- src/libexpr/flake/flakeref.cc | 279 +++++++++++++++++++--------------- 1 file changed, 155 insertions(+), 124 deletions(-) diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 2155c5f12..f7ef19b3f 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -69,127 +69,130 @@ std::optional maybeParseFlakeRef( } } -std::pair parseFlakeRefWithFragment( +std::pair parsePathFlakeRefWithFragment( const std::string & url, const std::optional & baseDir, bool allowMissing, bool isFlake) { - using namespace fetchers; + std::string path = url; + std::string fragment = ""; + std::map query; + auto pathEnd = url.find_first_of("#?"); + auto fragmentStart = pathEnd; + if (pathEnd != std::string::npos && url[pathEnd] == '?') { + fragmentStart = url.find("#"); + } + if (pathEnd != std::string::npos) { + path = url.substr(0, pathEnd); + } + if (fragmentStart != std::string::npos) { + fragment = percentDecode(url.substr(fragmentStart+1)); + } + if (pathEnd != std::string::npos && fragmentStart != std::string::npos) { + query = decodeQuery(url.substr(pathEnd+1, fragmentStart)); + } + + if (baseDir) { + /* Check if 'url' is a path (either absolute or relative + to 'baseDir'). If so, search upward to the root of the + repo (i.e. the directory containing .git). */ + + path = absPath(path, baseDir); + + if (isFlake) { + + if (!allowMissing && !pathExists(path + "/flake.nix")){ + notice("path '%s' does not contain a 'flake.nix', searching up",path); + + // Save device to detect filesystem boundary + dev_t device = lstat(path).st_dev; + bool found = false; + while (path != "/") { + if (pathExists(path + "/flake.nix")) { + found = true; + break; + } else if (pathExists(path + "/.git")) + throw Error("path '%s' is not part of a flake (neither it nor its parent directories contain a 'flake.nix' file)", path); + else { + if (lstat(path).st_dev != device) + throw Error("unable to find a flake before encountering filesystem boundary at '%s'", path); + } + path = dirOf(path); + } + if (!found) + throw BadURL("could not find a flake.nix file"); + } + + if (!S_ISDIR(lstat(path).st_mode)) + throw BadURL("path '%s' is not a flake (because it's not a directory)", path); + + if (!allowMissing && !pathExists(path + "/flake.nix")) + throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path); + + auto flakeRoot = path; + std::string subdir; + + while (flakeRoot != "/") { + if (pathExists(flakeRoot + "/.git")) { + auto base = std::string("git+file://") + flakeRoot; + + auto parsedURL = ParsedURL{ + .url = base, // FIXME + .base = base, + .scheme = "git+file", + .authority = "", + .path = flakeRoot, + .query = query, + }; + + if (subdir != "") { + if (parsedURL.query.count("dir")) + throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url); + parsedURL.query.insert_or_assign("dir", subdir); + } + + if (pathExists(flakeRoot + "/.git/shallow")) + parsedURL.query.insert_or_assign("shallow", "1"); + + return std::make_pair( + FlakeRef(fetchers::Input::fromURL(parsedURL), getOr(parsedURL.query, "dir", "")), + fragment); + } + + subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir); + flakeRoot = dirOf(flakeRoot); + } + } + + } else { + if (!hasPrefix(path, "/")) + throw BadURL("flake reference '%s' is not an absolute path", url); + path = canonPath(path + "/" + getOr(query, "dir", "")); + } + + fetchers::Attrs attrs; + attrs.insert_or_assign("type", "path"); + attrs.insert_or_assign("path", path); + + return std::make_pair(FlakeRef(fetchers::Input::fromAttrs(std::move(attrs)), ""), fragment); +}; + + +/* Check if 'url' is a flake ID. This is an abbreviated syntax for + 'flake:?ref=&rev='. */ +std::optional> parseFlakeIdRef( + const std::string & url, + bool isFlake +) +{ + std::smatch match; static std::regex flakeRegex( "((" + flakeIdRegexS + ")(?:/(?:" + refAndOrRevRegex + "))?)" + "(?:#(" + queryRegex + "))?", std::regex::ECMAScript); - std::smatch match; - - auto parsePathFlakeRef = [&]() { - std::string path = url; - std::string fragment = ""; - std::map query = {}; - auto pathEnd = url.find_first_of("#?"); - auto fragmentStart = pathEnd; - if (pathEnd != std::string::npos && url[pathEnd] == '?') - fragmentStart = url.find("#"); - if (pathEnd != std::string::npos) { - path = url.substr(0, pathEnd); - } - if (fragmentStart != std::string::npos) { - fragment = percentDecode(url.substr(fragmentStart+1)); - } - if (fragmentStart != std::string::npos && pathEnd != std::string::npos) { - query = decodeQuery(url.substr(pathEnd+1, fragmentStart)); - } - - if (baseDir) { - /* Check if 'url' is a path (either absolute or relative - to 'baseDir'). If so, search upward to the root of the - repo (i.e. the directory containing .git). */ - - path = absPath(path, baseDir); - - if (isFlake) { - - if (!allowMissing && !pathExists(path + "/flake.nix")){ - notice("path '%s' does not contain a 'flake.nix', searching up",path); - - // Save device to detect filesystem boundary - dev_t device = lstat(path).st_dev; - bool found = false; - while (path != "/") { - if (pathExists(path + "/flake.nix")) { - found = true; - break; - } else if (pathExists(path + "/.git")) - throw Error("path '%s' is not part of a flake (neither it nor its parent directories contain a 'flake.nix' file)", path); - else { - if (lstat(path).st_dev != device) - throw Error("unable to find a flake before encountering filesystem boundary at '%s'", path); - } - path = dirOf(path); - } - if (!found) - throw BadURL("could not find a flake.nix file"); - } - - if (!S_ISDIR(lstat(path).st_mode)) - throw BadURL("path '%s' is not a flake (because it's not a directory)", path); - - if (!allowMissing && !pathExists(path + "/flake.nix")) - throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path); - - auto flakeRoot = path; - std::string subdir; - - while (flakeRoot != "/") { - if (pathExists(flakeRoot + "/.git")) { - auto base = std::string("git+file://") + flakeRoot; - - auto parsedURL = ParsedURL{ - .url = base, // FIXME - .base = base, - .scheme = "git+file", - .authority = "", - .path = flakeRoot, - .query = query, - }; - - if (subdir != "") { - if (parsedURL.query.count("dir")) - throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url); - parsedURL.query.insert_or_assign("dir", subdir); - } - - if (pathExists(flakeRoot + "/.git/shallow")) - parsedURL.query.insert_or_assign("shallow", "1"); - - return std::make_pair( - FlakeRef(Input::fromURL(parsedURL, isFlake), getOr(parsedURL.query, "dir", "")), - fragment); - } - - subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir); - flakeRoot = dirOf(flakeRoot); - } - } - - } else { - if (!hasPrefix(path, "/")) - throw BadURL("flake reference '%s' is not an absolute path", url); - path = canonPath(path + "/" + getOr(query, "dir", "")); - } - - fetchers::Attrs attrs; - attrs.insert_or_assign("type", "path"); - attrs.insert_or_assign("path", path); - - return std::make_pair(FlakeRef(Input::fromAttrs(std::move(attrs)), ""), fragment); - }; - - /* Check if 'url' is a flake ID. This is an abbreviated syntax for - 'flake:?ref=&rev='. */ - if (std::regex_match(url, match, flakeRegex)) { auto parsedURL = ParsedURL{ .url = url, @@ -200,25 +203,53 @@ std::pair parseFlakeRefWithFragment( }; return std::make_pair( - FlakeRef(Input::fromURL(parsedURL), ""), + FlakeRef(fetchers::Input::fromURL(parsedURL, isFlake), ""), percentDecode(match.str(6))); } - else { - try { - auto parsedURL = parseURL(url); - std::string fragment; - std::swap(fragment, parsedURL.fragment); + return {}; +} - auto input = Input::fromURL(parsedURL, isFlake); - input.parent = baseDir; +std::optional> parseURLFlakeRef( + const std::string & url, + const std::optional & baseDir, + bool isFlake +) +{ + ParsedURL parsedURL; + try { + parsedURL = parseURL(url); + } catch (BadURL &) { + return std::nullopt; + } - return std::make_pair( - FlakeRef(std::move(input), getOr(parsedURL.query, "dir", "")), - fragment); - } catch (BadURL &) { - return parsePathFlakeRef(); - } + std::string fragment; + std::swap(fragment, parsedURL.fragment); + + auto input = fetchers::Input::fromURL(parsedURL, isFlake); + input.parent = baseDir; + + return std::make_pair( + FlakeRef(std::move(input), getOr(parsedURL.query, "dir", "")), + fragment); +} + +std::pair parseFlakeRefWithFragment( + const std::string & url, + const std::optional & baseDir, + bool allowMissing, + bool isFlake) +{ + using namespace fetchers; + + std::smatch match; + + if (auto res = parseFlakeIdRef(url, isFlake)) { + return *res; + } else if (auto res = parseURLFlakeRef(url, baseDir, isFlake)) { + return *res; + } else { + return parsePathFlakeRefWithFragment(url, baseDir, allowMissing, isFlake); } } From 34115076969f20be935ae8eaf579928ac07981e3 Mon Sep 17 00:00:00 2001 From: Rasmus Rendal Date: Tue, 14 Jun 2022 17:38:48 +0200 Subject: [PATCH 104/139] Document the percent-encoding mechanism --- doc/manual/src/release-notes/rl-next.md | 4 ++++ src/nix/flake.md | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index c869b5e2f..2467e7ccc 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1 +1,5 @@ # Release X.Y (202?-??-??) + +- [URL flake references](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) now support [percent-encoded](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1) characters. + +- [Path-like flake references](@docroot@/command-ref/new-cli/nix3-flake.md#path-like-syntax) now accept arbitrary unicode characters (except `#` and `?`). diff --git a/src/nix/flake.md b/src/nix/flake.md index 92f477917..1c512b315 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -67,6 +67,13 @@ inputs.nixpkgs = { }; ``` +Following [RFC 3986](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1), +characters outside of the allowed range (i.e. are neither [reserved +character](https://datatracker.ietf.org/doc/html/rfc3986#section-2.2) or +[unreserved +characters](https://datatracker.ietf.org/doc/html/rfc3986#section-2.3)) must be +percent-encoded. + ### Examples Here are some examples of flake references in their URL-like representation: @@ -103,10 +110,15 @@ The semantic of such a path is as follows: 2. The filesystem root (/), or 3. A folder on a different mount point. +Contrary to URL-like reference, path-like flake references can contain +arbitrary unicode characters (except `#` and `?`). + ### Examples * `.`: The flake to which the current directory belongs to. * `/home/alice/src/patchelf`: A flake in some other directory. +* `./../sub directory/with Ûñî©ôδ€`: A flake in another relative directory that + has Unicode characters in its name. ## Flake reference attributes From 39ba81a4eb7404f9a12cf0768da51c0e0e26b588 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 7 Sep 2023 22:52:57 -0400 Subject: [PATCH 105/139] Improve internal API docs for two file hashing functions Co-Authored-By: Matthew Bauer Co-Authored-By: Carlo Nucera --- src/libutil/hash.hh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index ae3ee40f4..c3aa5cd81 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -145,13 +145,17 @@ std::string printHash16or32(const Hash & hash); Hash hashString(HashType ht, std::string_view s); /** - * Compute the hash of the given file. + * Compute the hash of the given file, hashing its contents directly. + * + * (Metadata, such as the executable permission bit, is ignored.) */ Hash hashFile(HashType ht, const Path & path); /** - * Compute the hash of the given path. The hash is defined as - * (essentially) hashString(ht, dumpPath(path)). + * Compute the hash of the given path, serializing as a Nix Archive and + * then hashing that. + * + * The hash is defined as (essentially) hashString(ht, dumpPath(path)). */ typedef std::pair HashResult; HashResult hashPath(HashType ht, const Path & path, From 9d6114313b1b06a020bed230cfe7f05e6075b875 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 7 Sep 2023 22:31:19 -0400 Subject: [PATCH 106/139] Move `ParseSink` to its own header We will soon add a new implemenation so the one for NARs in `archive.cc` isn't the only one. Co-Authored-By: Matthew Bauer Co-Authored-By: Carlo Nucera --- src/libutil/archive.hh | 17 +---------------- src/libutil/fs-sink.hh | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 16 deletions(-) create mode 100644 src/libutil/fs-sink.hh diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 2cf164a41..3530783c1 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -3,6 +3,7 @@ #include "types.hh" #include "serialise.hh" +#include "fs-sink.hh" namespace nix { @@ -72,22 +73,6 @@ time_t dumpPathAndGetMtime(const Path & path, Sink & sink, */ void dumpString(std::string_view s, Sink & sink); -/** - * \todo Fix this API, it sucks. - */ -struct ParseSink -{ - virtual void createDirectory(const Path & path) { }; - - virtual void createRegularFile(const Path & path) { }; - virtual void closeRegularFile() { }; - virtual void isExecutable() { }; - virtual void preallocateContents(uint64_t size) { }; - virtual void receiveContents(std::string_view data) { }; - - virtual void createSymlink(const Path & path, const std::string & target) { }; -}; - /** * If the NAR archive contains a single file at top-level, then save * the contents of the file to `s`. Otherwise barf. diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh new file mode 100644 index 000000000..ed6484e61 --- /dev/null +++ b/src/libutil/fs-sink.hh @@ -0,0 +1,25 @@ +#pragma once +///@file + +#include "types.hh" +#include "serialise.hh" + +namespace nix { + +/** + * \todo Fix this API, it sucks. + */ +struct ParseSink +{ + virtual void createDirectory(const Path & path) { }; + + virtual void createRegularFile(const Path & path) { }; + virtual void closeRegularFile() { }; + virtual void isExecutable() { }; + virtual void preallocateContents(uint64_t size) { }; + virtual void receiveContents(std::string_view data) { }; + + virtual void createSymlink(const Path & path, const std::string & target) { }; +}; + +} From 8a416e819c9727c2e9efd70f1d2a4b0bfda11663 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 7 Sep 2023 22:49:01 -0400 Subject: [PATCH 107/139] Move `RestoreSink` to `fs-sink.cc` Co-Authored-By: Matthew Bauer Co-Authored-By: Carlo Nucera --- src/libutil/archive.cc | 67 ---------------------------------- src/libutil/fs-sink.cc | 82 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 67 deletions(-) create mode 100644 src/libutil/fs-sink.cc diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 268a798d9..0f0cae7c0 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -27,8 +27,6 @@ struct ArchiveSettings : Config #endif "use-case-hack", "Whether to enable a Darwin-specific hack for dealing with file name collisions."}; - Setting preallocateContents{this, false, "preallocate-contents", - "Whether to preallocate files when writing objects with known size."}; }; static ArchiveSettings archiveSettings; @@ -302,71 +300,6 @@ void parseDump(ParseSink & sink, Source & source) } -struct RestoreSink : ParseSink -{ - Path dstPath; - AutoCloseFD fd; - - void createDirectory(const Path & path) override - { - Path p = dstPath + path; - if (mkdir(p.c_str(), 0777) == -1) - throw SysError("creating directory '%1%'", p); - }; - - void createRegularFile(const Path & path) override - { - Path p = dstPath + path; - fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); - if (!fd) throw SysError("creating file '%1%'", p); - } - - void closeRegularFile() override - { - /* Call close explicitly to make sure the error is checked */ - fd.close(); - } - - void isExecutable() override - { - struct stat st; - if (fstat(fd.get(), &st) == -1) - throw SysError("fstat"); - if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) - throw SysError("fchmod"); - } - - void preallocateContents(uint64_t len) override - { - if (!archiveSettings.preallocateContents) - return; - -#if HAVE_POSIX_FALLOCATE - if (len) { - errno = posix_fallocate(fd.get(), 0, len); - /* Note that EINVAL may indicate that the underlying - filesystem doesn't support preallocation (e.g. on - OpenSolaris). Since preallocation is just an - optimisation, ignore it. */ - if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) - throw SysError("preallocating file of %1% bytes", len); - } -#endif - } - - void receiveContents(std::string_view data) override - { - writeFull(fd.get(), data); - } - - void createSymlink(const Path & path, const std::string & target) override - { - Path p = dstPath + path; - nix::createSymlink(target, p); - } -}; - - void restorePath(const Path & path, Source & source) { RestoreSink sink; diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc new file mode 100644 index 000000000..fcffd3216 --- /dev/null +++ b/src/libutil/fs-sink.cc @@ -0,0 +1,82 @@ +#include + +#include "config.hh" +#include "fs-sink.hh" + +namespace nix { + +struct RestoreSinkSettings : Config +{ + Setting preallocateContents{this, false, "preallocate-contents", + "Whether to preallocate files when writing objects with known size."}; +}; + +static RestoreSinkSettings restoreSinkSettings; + +static GlobalConfig::Register r1(&restoreSinkSettings); + +struct RestoreSink : ParseSink +{ + Path dstPath; + AutoCloseFD fd; + + void createDirectory(const Path & path) override + { + Path p = dstPath + path; + if (mkdir(p.c_str(), 0777) == -1) + throw SysError("creating directory '%1%'", p); + }; + + void createRegularFile(const Path & path) override + { + Path p = dstPath + path; + fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); + if (!fd) throw SysError("creating file '%1%'", p); + } + + void closeRegularFile() override + { + /* Call close explicitly to make sure the error is checked */ + fd.close(); + } + + void isExecutable() override + { + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError("fstat"); + if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) + throw SysError("fchmod"); + } + + void preallocateContents(uint64_t len) override + { + if (!archiveSettings.preallocateContents) + return; + +#if HAVE_POSIX_FALLOCATE + if (len) { + errno = posix_fallocate(fd.get(), 0, len); + /* Note that EINVAL may indicate that the underlying + filesystem doesn't support preallocation (e.g. on + OpenSolaris). Since preallocation is just an + optimisation, ignore it. */ + if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) + throw SysError("preallocating file of %1% bytes", len); + } +#endif + } + + void receiveContents(std::string_view data) override + { + writeFull(fd.get(), data); + } + + void createSymlink(const Path & path, const std::string & target) override + { + Path p = dstPath + path; + nix::createSymlink(target, p); + } +}; + +} From f2e201fbdb93925b9588f78a03d67fcd91c74b65 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 7 Sep 2023 22:49:01 -0400 Subject: [PATCH 108/139] Expose `RestoreSink` in header (`fs-sink.hh`) Co-Authored-By: Matthew Bauer Co-Authored-By: Carlo Nucera --- src/libutil/fs-sink.cc | 117 ++++++++++++++++++++--------------------- src/libutil/fs-sink.hh | 17 ++++++ 2 files changed, 73 insertions(+), 61 deletions(-) diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc index fcffd3216..a08a723a4 100644 --- a/src/libutil/fs-sink.cc +++ b/src/libutil/fs-sink.cc @@ -15,68 +15,63 @@ static RestoreSinkSettings restoreSinkSettings; static GlobalConfig::Register r1(&restoreSinkSettings); -struct RestoreSink : ParseSink + +void RestoreSink::createDirectory(const Path & path) { - Path dstPath; - AutoCloseFD fd; - - void createDirectory(const Path & path) override - { - Path p = dstPath + path; - if (mkdir(p.c_str(), 0777) == -1) - throw SysError("creating directory '%1%'", p); - }; - - void createRegularFile(const Path & path) override - { - Path p = dstPath + path; - fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); - if (!fd) throw SysError("creating file '%1%'", p); - } - - void closeRegularFile() override - { - /* Call close explicitly to make sure the error is checked */ - fd.close(); - } - - void isExecutable() override - { - struct stat st; - if (fstat(fd.get(), &st) == -1) - throw SysError("fstat"); - if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) - throw SysError("fchmod"); - } - - void preallocateContents(uint64_t len) override - { - if (!archiveSettings.preallocateContents) - return; - -#if HAVE_POSIX_FALLOCATE - if (len) { - errno = posix_fallocate(fd.get(), 0, len); - /* Note that EINVAL may indicate that the underlying - filesystem doesn't support preallocation (e.g. on - OpenSolaris). Since preallocation is just an - optimisation, ignore it. */ - if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) - throw SysError("preallocating file of %1% bytes", len); - } -#endif - } - - void receiveContents(std::string_view data) override - { - writeFull(fd.get(), data); - } - - void createSymlink(const Path & path, const std::string & target) override - { - Path p = dstPath + path; - nix::createSymlink(target, p); - } + Path p = dstPath + path; + if (mkdir(p.c_str(), 0777) == -1) + throw SysError("creating directory '%1%'", p); }; +void RestoreSink::createRegularFile(const Path & path) +{ + Path p = dstPath + path; + fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); + if (!fd) throw SysError("creating file '%1%'", p); +} + +void RestoreSink::closeRegularFile() +{ + /* Call close explicitly to make sure the error is checked */ + fd.close(); +} + +void RestoreSink::isExecutable() +{ + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError("fstat"); + if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) + throw SysError("fchmod"); +} + +void RestoreSink::preallocateContents(uint64_t len) +{ + if (!restoreSinkSettings.preallocateContents) + return; + +#if HAVE_POSIX_FALLOCATE + if (len) { + errno = posix_fallocate(fd.get(), 0, len); + /* Note that EINVAL may indicate that the underlying + filesystem doesn't support preallocation (e.g. on + OpenSolaris). Since preallocation is just an + optimisation, ignore it. */ + if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) + throw SysError("preallocating file of %1% bytes", len); + } +#endif +} + +void RestoreSink::receiveContents(std::string_view data) +{ + writeFull(fd.get(), data); +} + +void RestoreSink::createSymlink(const Path & path, const std::string & target) +{ + Path p = dstPath + path; + nix::createSymlink(target, p); +} + } diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh index ed6484e61..6837e2fc4 100644 --- a/src/libutil/fs-sink.hh +++ b/src/libutil/fs-sink.hh @@ -22,4 +22,21 @@ struct ParseSink virtual void createSymlink(const Path & path, const std::string & target) { }; }; +struct RestoreSink : ParseSink +{ + Path dstPath; + AutoCloseFD fd; + + + void createDirectory(const Path & path) override; + + void createRegularFile(const Path & path) override; + void closeRegularFile() override; + void isExecutable() override; + void preallocateContents(uint64_t size) override; + void receiveContents(std::string_view data) override; + + void createSymlink(const Path & path, const std::string & target) override; +}; + } From 694810ba34a74226eba22daf5c6264b9d81e2926 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 22 Sep 2023 23:54:04 -0400 Subject: [PATCH 109/139] doc: `showOptions`: Move union to caller `showOptions` itself doesn't care, so it shouldn't take two separate arguments. --- doc/manual/generate-manpage.nix | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 87891fa7a..b85c488c9 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -78,16 +78,15 @@ let maybeOptions = optionalString (details.flags != {}) '' # Options - ${showOptions details.flags toplevel.flags} + ${showOptions (details.flags // toplevel.flags)} > **Note** > > See [`man nix.conf`](@docroot@/command-ref/conf-file.md#command-line-flags) for overriding configuration settings with command line flags. ''; - showOptions = options: commonOptions: + showOptions = allOptions: let - allOptions = options // commonOptions; showCategory = cat: '' ${optionalString (cat != "") "**${cat}:**"} From 9c640c122931aa6515e1f3c80cb4d415a1352cc7 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 23 Sep 2023 00:28:16 -0400 Subject: [PATCH 110/139] doc: `showOptions`: Simplify code with `builtins.groupBy` This makes grouping options by category much nicer. No behavior should be changed. --- doc/manual/generate-manpage.nix | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index b85c488c9..e18810594 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -1,8 +1,8 @@ let inherit (builtins) - attrNames attrValues fromJSON listToAttrs mapAttrs + attrNames attrValues fromJSON listToAttrs mapAttrs groupBy concatStringsSep concatMap length lessThan replaceStrings sort; - inherit (import ./utils.nix) concatStrings optionalString filterAttrs trim squash unique showSettings; + inherit (import ./utils.nix) attrsToList concatStrings optionalString filterAttrs trim squash unique showSettings; in commandDump: @@ -87,12 +87,11 @@ let showOptions = allOptions: let - showCategory = cat: '' + showCategory = cat: opts: '' ${optionalString (cat != "") "**${cat}:**"} - ${listOptions (filterAttrs (n: v: v.category == cat) allOptions)} + ${concatStringsSep "\n" (attrValues (mapAttrs showOption opts))} ''; - listOptions = opts: concatStringsSep "\n" (attrValues (mapAttrs showOption opts)); showOption = name: option: let shortName = optionalString @@ -106,8 +105,12 @@ let ${option.description} ''; - categories = sort lessThan (unique (map (cmd: cmd.category) (attrValues allOptions))); - in concatStrings (map showCategory categories); + categories = mapAttrs + (_: listToAttrs) + (groupBy + (cmd: cmd.value.category) + (attrsToList allOptions)); + in concatStrings (attrValues (mapAttrs showCategory categories)); in squash result; appendName = filename: name: (if filename == "nix" then "nix3" else filename) + "-" + name; From 1d9fd3a6f8ac53be4ba7d409defe291c0dbdf97a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 22 Sep 2023 23:55:06 -0400 Subject: [PATCH 111/139] manual / manpages: Adjust option filter filtering, move from C++ to Nix Behavior change: Before we only showed uption if the command-specific options were non-empty. But that is somewhat odd since we also show common options. Now, we do everything based on the union of both sorts of options (with hidden-categories filtered, as before). Implementation change: The JSON dumping once again includes all options; the filtering of hidden categories is done in the Nix instead. This is better separation of "content" vs "presentation", and prepare the way for the HTML manual vs manpages / `--help` doing different things. --- doc/manual/generate-manpage.nix | 8 ++++++-- src/libutil/args.cc | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index e18810594..b41730bc2 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -75,10 +75,14 @@ let (details ? doc) (replaceStrings ["@stores@"] [storeDocs] details.doc); - maybeOptions = optionalString (details.flags != {}) '' + maybeOptions = let + allVisibleOptions = filterAttrs + (_: o: ! o.hiddenCategory) + (details.flags // toplevel.flags); + in optionalString (allVisibleOptions != {}) '' # Options - ${showOptions (details.flags // toplevel.flags)} + ${showOptions allVisibleOptions} > **Note** > diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 8db293762..e410c7eec 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -236,7 +236,7 @@ nlohmann::json Args::toJSON() for (auto & [name, flag] : longFlags) { auto j = nlohmann::json::object(); - if (hiddenCategories.count(flag->category)) continue; + j["hiddenCategory"] = hiddenCategories.count(flag->category) > 0; if (flag->aliases.count(name)) continue; if (flag->shortName) j["shortName"] = std::string(1, flag->shortName); From 9f93972c4d73abdcde789112229209af2cbda520 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 23 Sep 2023 00:35:03 -0400 Subject: [PATCH 112/139] manual / manpages: Make option category names a proper subheader Before they were an "ad-hoc" header with bold and a colon; now they are a proper subheader. For the man pages, this doesn't make much of a difference, but it will help more on for the HTML manual, where things can be restyled. Again, good separation of content vs presentation. --- doc/manual/generate-manpage.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index b41730bc2..21a4802e5 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -92,7 +92,7 @@ let showOptions = allOptions: let showCategory = cat: opts: '' - ${optionalString (cat != "") "**${cat}:**"} + ${optionalString (cat != "") "## ${cat}"} ${concatStringsSep "\n" (attrValues (mapAttrs showOption opts))} ''; From e4b83fbfe2e08ddd2ebd409c26f2dc4c11ee1d4a Mon Sep 17 00:00:00 2001 From: Yingchi Long Date: Sun, 24 Sep 2023 14:54:41 +0800 Subject: [PATCH 113/139] libexpr: const rvalue reference -> value for nix::Expr nodes --- src/libexpr/nixexpr.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 5ca3d1fa6..944977e50 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -238,7 +238,7 @@ struct ExprSelect : Expr PosIdx pos; Expr * e, * def; AttrPath attrPath; - ExprSelect(const PosIdx & pos, Expr * e, const AttrPath && attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(std::move(attrPath)) { }; + ExprSelect(const PosIdx & pos, Expr * e, AttrPath attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(std::move(attrPath)) { }; ExprSelect(const PosIdx & pos, Expr * e, Symbol name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); }; PosIdx getPos() const override { return pos; } COMMON_METHODS @@ -248,7 +248,7 @@ struct ExprOpHasAttr : Expr { Expr * e; AttrPath attrPath; - ExprOpHasAttr(Expr * e, const AttrPath && attrPath) : e(e), attrPath(std::move(attrPath)) { }; + ExprOpHasAttr(Expr * e, AttrPath attrPath) : e(e), attrPath(std::move(attrPath)) { }; PosIdx getPos() const override { return e->getPos(); } COMMON_METHODS }; From 89e5e68799132bc2f1d2551002dab84b63fde9ac Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sun, 24 Sep 2023 20:28:34 +0200 Subject: [PATCH 114/139] doc/hacking: fix `make` target to build the docs (#9033) Was confused why `make html` didn't work while working on #9032, but then I realized that after this section was written, the target was renamed to `manual-html` in 6910f5dcb6784360589b860b8f80487a5c97fe08. --- doc/manual/src/contributing/hacking.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 22f629c6f..a2446b53a 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -228,7 +228,7 @@ This happens late in the process, so `nix build` is not suitable for iterating. To build the manual incrementally, run: ```console -make html -j $NIX_BUILD_CORES +make manual-html -j $NIX_BUILD_CORES ``` In order to reflect changes to the [Makefile], clear all generated files before re-building: From b3433099d4ef3fbc2ef86f97cf19dcd543bd05e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophane=20Hufschmitt?= <7226587+thufschmitt@users.noreply.github.com> Date: Mon, 25 Sep 2023 09:56:49 +0200 Subject: [PATCH 115/139] Apply suggestions from code review Co-authored-by: Valentin Gagarin --- src/nix/flake.md | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/nix/flake.md b/src/nix/flake.md index 1c512b315..939269c6a 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -68,11 +68,9 @@ inputs.nixpkgs = { ``` Following [RFC 3986](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1), -characters outside of the allowed range (i.e. are neither [reserved -character](https://datatracker.ietf.org/doc/html/rfc3986#section-2.2) or -[unreserved -characters](https://datatracker.ietf.org/doc/html/rfc3986#section-2.3)) must be -percent-encoded. +characters outside of the allowed range (i.e. neither [reserved characters](https://datatracker.ietf.org/doc/html/rfc3986#section-2.2) +nor [unreserved characters](https://datatracker.ietf.org/doc/html/rfc3986#section-2.3)) +must be percent-encoded. ### Examples @@ -110,8 +108,7 @@ The semantic of such a path is as follows: 2. The filesystem root (/), or 3. A folder on a different mount point. -Contrary to URL-like reference, path-like flake references can contain -arbitrary unicode characters (except `#` and `?`). +Contrary to URL-like references, path-like flake references can contain arbitrary unicode characters (except `#` and `?`). ### Examples From 4606a07bb62c650d2261f2d8d5e93e547acf0af6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 25 Sep 2023 08:20:39 -0400 Subject: [PATCH 116/139] generate-manpage.nix: Add comment explaining one bit --- doc/manual/generate-manpage.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 21a4802e5..41725aed6 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -110,6 +110,7 @@ let ${option.description} ''; categories = mapAttrs + # Convert each group from a list of key-value pairs back to an attrset (_: listToAttrs) (groupBy (cmd: cmd.value.category) From 70b5e6050cd135b75942f731ad892e4f473c6c21 Mon Sep 17 00:00:00 2001 From: waalge <47293755+waalge@users.noreply.github.com> Date: Mon, 25 Sep 2023 13:39:11 +0000 Subject: [PATCH 117/139] fix docstring --- src/libexpr/primops.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index cd9a05bb2..0d503ae26 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2991,7 +2991,7 @@ static RegisterPrimOp primop_tail({ .name = "__tail", .args = {"list"}, .doc = R"( - Return the second to last elements of a list; abort evaluation if + Return the list without its first item; abort evaluation if the argument isn’t a list or is an empty list. > **Warning** From ad213103d85d22b52c608362cb2127fa10e49620 Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Mon, 25 Sep 2023 17:45:13 +0100 Subject: [PATCH 118/139] src/libstore/profiles.cc: fix comparison of sign difference Detected by `gcc` as: CXX src/libstore/profiles.o src/libstore/profiles.cc: In function 'void nix::deleteGenerationsGreaterThan(const Path&, GenerationNumber, bool)': src/libstore/profiles.cc:186:50: warning: comparison of integer expressions of different signedness: 'int' and 'nix::GenerationNumber' {aka 'long unsigned int'} [-Wsign-compare] 186 | for (auto keep = 0; i != gens.rend() && keep < max; ++i, ++keep); | ~~~~~^~~~~ --- src/libstore/profiles.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 4e9955948..239047dd6 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -183,7 +183,7 @@ void deleteGenerationsGreaterThan(const Path & profile, GenerationNumber max, bo iterDropUntil(gens, i, [&](auto & g) { return g.number == curGen; }); // Skip over `max` generations, preserving them - for (auto keep = 0; i != gens.rend() && keep < max; ++i, ++keep); + for (GenerationNumber keep = 0; i != gens.rend() && keep < max; ++i, ++keep); // Delete the rest for (; i != gens.rend(); ++i) From bd24176ac57b9d418b4ec51448b188cb74b87945 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 25 Sep 2023 17:51:17 +0100 Subject: [PATCH 119/139] libexpr/nixexpr.hh: Remove redundant inline This is redundant since definitions in C++ record are implicitly inline-ed. Co-authored-by: Yingchi Long --- src/libexpr/nixexpr.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 17b2d1136..5c9242759 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -156,7 +156,7 @@ std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath) struct Expr { static unsigned long nrExprs; - inline Expr() { + Expr() { nrExprs++; } virtual ~Expr() { }; From cba53b3a18c2758e3d969d9d5781294560afe1b7 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 26 Sep 2023 00:39:11 +0200 Subject: [PATCH 120/139] reformat for readability --- doc/manual/src/command-ref/nix-store/realise.md | 8 ++++++-- doc/manual/src/glossary.md | 10 +++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index a421a4220..5428d57fa 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -15,8 +15,12 @@ Each of *paths* is processed as follows: 1. If it is not [valid], substitute the store derivation file itself. 2. Realise its [output paths]: - Try to fetch from [substituters] the [store objects] associated with the output paths in the store derivation's [closure]. - - With [content-addressed derivations] (experimental): Determine the output paths to realise by querying content-addressed realisation entries in the [Nix database]. - - For any store paths that cannot be substituted, produce the required store objects. This involves first realising all outputs of the derivation's dependencies and then running the derivation's [`builder`](@docroot@/language/derivations.md#attr-builder) executable. + - With [content-addressed derivations] (experimental): + Determine the output paths to realise by querying content-addressed realisation entries in the [Nix database]. + - For any store paths that cannot be substituted, produce the required store objects: + 1. Realise all outputs of the derivation's dependencies + 2. Run the derivation's [`builder`](@docroot@/language/derivations.md#attr-builder) executable + - Otherwise, and if the path is not already valid: Try to fetch the associated [store objects] in the path's [closure] from [substituters]. If no substitutes are available and no store derivation is given, realisation fails. diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index b7c340d36..5345f2deb 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -33,11 +33,15 @@ Ensure a [store path] is [valid][validity]. - This means either running the [`builder`](@docroot@/language/derivations.md#attr-builder) executable as specified in the corresponding [derivation], or fetching a pre-built [store object] from a [substituter], or delegating to a [remote builder](@docroot@/advanced-topics/distributed-builds.html) and retrieving the outputs. + This can be achieved by: + - Fetching a pre-built [store object] from a [substituter] + - Running the [`builder`](@docroot@/language/derivations.md#attr-builder) executable as specified in the corresponding [derivation] + - Delegating to a [remote builder](@docroot@/advanced-topics/distributed-builds.html) and retrieving the outputs + - See [`nix-build`](./command-ref/nix-build.md) and [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md). + See [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md) for a detailed description of the algorithm. - See [`nix build`](./command-ref/new-cli/nix3-build.md) (experimental). + See also [`nix-build`](./command-ref/nix-build.md) and [`nix build`](./command-ref/new-cli/nix3-build.md) (experimental). [realise]: #gloss-realise From 541890463d83d1a818dfabc79053d1302bd828a2 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 18 Jul 2023 13:49:59 +0200 Subject: [PATCH 121/139] make separate section for builder execution --- doc/manual/src/language/derivations.md | 116 +++++++++++++------------ 1 file changed, 59 insertions(+), 57 deletions(-) diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index 79a09122a..df4673cf5 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -93,69 +93,71 @@ wrapper around `derivation` that adds a default value for `system` and always uses Bash as the builder, to which the supplied builder is passed as a command-line argument. See the Nixpkgs manual for details. -The builder is executed as follows: +## Builder execution - - A temporary directory is created under the directory specified by - `TMPDIR` (default `/tmp`) where the build will take place. The - current directory is changed to this directory. +The [`builder`](#attr-builder) is executed as follows: - - The environment is cleared and set to the derivation attributes, as - specified above. +- A temporary directory is created under the directory specified by + `TMPDIR` (default `/tmp`) where the build will take place. The + current directory is changed to this directory. - - In addition, the following variables are set: - - - `NIX_BUILD_TOP` contains the path of the temporary directory for - this build. - - - Also, `TMPDIR`, `TEMPDIR`, `TMP`, `TEMP` are set to point to the - temporary directory. This is to prevent the builder from - accidentally writing temporary files anywhere else. Doing so - might cause interference by other processes. - - - `PATH` is set to `/path-not-set` to prevent shells from - initialising it to their built-in default value. - - - `HOME` is set to `/homeless-shelter` to prevent programs from - using `/etc/passwd` or the like to find the user's home - directory, which could cause impurity. Usually, when `HOME` is - set, it is used as the location of the home directory, even if - it points to a non-existent path. - - - `NIX_STORE` is set to the path of the top-level Nix store - directory (typically, `/nix/store`). - - - For each output declared in `outputs`, the corresponding - environment variable is set to point to the intended path in the - Nix store for that output. Each output path is a concatenation - of the cryptographic hash of all build inputs, the `name` - attribute and the output name. (The output name is omitted if - it’s `out`.) +- The environment is cleared and set to the derivation attributes, as + specified above. - - If an output path already exists, it is removed. Also, locks are - acquired to prevent multiple Nix instances from performing the same - build at the same time. +- In addition, the following variables are set: - - A log of the combined standard output and error is written to - `/nix/var/log/nix`. + - `NIX_BUILD_TOP` contains the path of the temporary directory for + this build. - - The builder is executed with the arguments specified by the - attribute `args`. If it exits with exit code 0, it is considered to - have succeeded. + - Also, `TMPDIR`, `TEMPDIR`, `TMP`, `TEMP` are set to point to the + temporary directory. This is to prevent the builder from + accidentally writing temporary files anywhere else. Doing so + might cause interference by other processes. - - The temporary directory is removed (unless the `-K` option was - specified). + - `PATH` is set to `/path-not-set` to prevent shells from + initialising it to their built-in default value. - - If the build was successful, Nix scans each output path for - references to input paths by looking for the hash parts of the input - paths. Since these are potential runtime dependencies, Nix registers - them as dependencies of the output paths. + - `HOME` is set to `/homeless-shelter` to prevent programs from + using `/etc/passwd` or the like to find the user's home + directory, which could cause impurity. Usually, when `HOME` is + set, it is used as the location of the home directory, even if + it points to a non-existent path. - - After the build, Nix sets the last-modified timestamp on all files - in the build result to 1 (00:00:01 1/1/1970 UTC), sets the group to - the default group, and sets the mode of the file to 0444 or 0555 - (i.e., read-only, with execute permission enabled if the file was - originally executable). Note that possible `setuid` and `setgid` - bits are cleared. Setuid and setgid programs are not currently - supported by Nix. This is because the Nix archives used in - deployment have no concept of ownership information, and because it - makes the build result dependent on the user performing the build. + - `NIX_STORE` is set to the path of the top-level Nix store + directory (typically, `/nix/store`). + + - For each output declared in `outputs`, the corresponding + environment variable is set to point to the intended path in the + Nix store for that output. Each output path is a concatenation + of the cryptographic hash of all build inputs, the `name` + attribute and the output name. (The output name is omitted if + it’s `out`.) + +- If an output path already exists, it is removed. Also, locks are + acquired to prevent multiple Nix instances from performing the same + build at the same time. + +- A log of the combined standard output and error is written to + `/nix/var/log/nix`. + +- The builder is executed with the arguments specified by the + attribute `args`. If it exits with exit code 0, it is considered to + have succeeded. + +- The temporary directory is removed (unless the `-K` option was + specified). + +- If the build was successful, Nix scans each output path for + references to input paths by looking for the hash parts of the input + paths. Since these are potential runtime dependencies, Nix registers + them as dependencies of the output paths. + +- After the build, Nix sets the last-modified timestamp on all files + in the build result to 1 (00:00:01 1/1/1970 UTC), sets the group to + the default group, and sets the mode of the file to 0444 or 0555 + (i.e., read-only, with execute permission enabled if the file was + originally executable). Note that possible `setuid` and `setgid` + bits are cleared. Setuid and setgid programs are not currently + supported by Nix. This is because the Nix archives used in + deployment have no concept of ownership information, and because it + makes the build result dependent on the user performing the build. From e2f118efeda5c0a9b1380db736eb2b86f691b1fa Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 18 Jul 2023 14:03:17 +0200 Subject: [PATCH 122/139] restructure and reword input attributes section on `derivation` --- doc/manual/src/language/derivations.md | 208 +++++++++++++++---------- 1 file changed, 123 insertions(+), 85 deletions(-) diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index df4673cf5..56c6818ca 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -1,97 +1,135 @@ # Derivations -The most important built-in function is `derivation`, which is used to -describe a single derivation (a build task). It takes as input a set, -the attributes of which specify the inputs of the build. +The most important built-in function is `derivation`, which is used to describe a single derivation: +a build task to run a process on precisely defined input files and repeatably produce output files at uniquely determined file system paths. - - There must be an attribute named [`system`]{#attr-system} whose value must be a - string specifying a Nix system type, such as `"i686-linux"` or - `"x86_64-darwin"`. (To figure out your system type, run `nix -vv - --version`.) The build can only be performed on a machine and - operating system matching the system type. (Nix can automatically - [forward builds for other - platforms](../advanced-topics/distributed-builds.md) by forwarding - them to other machines.) +It takes as input an attribute set, the attributes of which specify the inputs to the build. +It outputs an attribute set, and produces a [store derivation](@docroot@/glossary.md#gloss-store-derivation) as a side effect of evaluation. - - There must be an attribute named `name` whose value must be a - string. This is used as a symbolic name for the package by - `nix-env`, and it is appended to the output paths of the derivation. + - - There must be an attribute named [`builder`]{#attr-builder} that identifies the - program that is executed to perform the build. It can be either a - derivation or a source (a local file reference, e.g., - `./builder.sh`). +## Input attributes - - Every attribute is passed as an environment variable to the builder. - Attribute values are translated to environment variables as follows: - - - Strings and numbers are just passed verbatim. - - - A *path* (e.g., `../foo/sources.tar`) causes the referenced file - to be copied to the store; its location in the store is put in - the environment variable. The idea is that all sources should - reside in the Nix store, since all inputs to a derivation should - reside in the Nix store. - - - A *derivation* causes that derivation to be built prior to the - present derivation; its default output path is put in the - environment variable. - - - Lists of the previous types are also allowed. They are simply - concatenated, separated by spaces. - - - `true` is passed as the string `1`, `false` and `null` are - passed as an empty string. +### Required - - The optional attribute `args` specifies command-line arguments to be - passed to the builder. It should be a list. +- [`name`]{#attr-name} ([String](@docroot@/language/values.md#type-string)) - - The optional attribute `outputs` specifies a list of symbolic - outputs of the derivation. By default, a derivation produces a - single output path, denoted as `out`. However, derivations can - produce multiple output paths. This is useful because it allows - outputs to be downloaded or garbage-collected separately. For - instance, imagine a library package that provides a dynamic library, - header files, and documentation. A program that links against the - library doesn’t need the header files and documentation at runtime, - and it doesn’t need the documentation at build time. Thus, the - library package could specify: - - ```nix - outputs = [ "lib" "headers" "doc" ]; - ``` - - This will cause Nix to pass environment variables `lib`, `headers` - and `doc` to the builder containing the intended store paths of each - output. The builder would typically do something like - - ```bash - ./configure \ - --libdir=$lib/lib \ - --includedir=$headers/include \ - --docdir=$doc/share/doc - ``` - - for an Autoconf-style package. You can refer to each output of a - derivation by selecting it as an attribute, e.g. - - ```nix - buildInputs = [ pkg.lib pkg.headers ]; - ``` - - The first element of `outputs` determines the *default output*. - Thus, you could also write - - ```nix - buildInputs = [ pkg pkg.headers ]; - ``` - - since `pkg` is equivalent to `pkg.lib`. + Symbolic name for the derivation. + It is appended to the [store paths](@docroot@/glossary.md#gloss-store-path) of the resulting store derivation and its [output paths][output path]. -The function `mkDerivation` in the Nixpkgs standard environment is a -wrapper around `derivation` that adds a default value for `system` and -always uses Bash as the builder, to which the supplied builder is passed -as a command-line argument. See the Nixpkgs manual for details. + Example: `name = "hello";` + +- [`system`]{#attr-system} ([String](@docroot@/language/values.md#type-string)) + + The system type on which the [`builder`](#attr-builder) executable can be run. + + Nix will only build derivations where the `system` attribute matches the current [`system` configuration option]. + It can automatically [build on other platforms](../advanced-topics/distributed-builds.md) by forwarding build requests to other machines. + + Examples: + + `system = "x86_64-linux";` + + `system = builtins.currentSystem;` + + [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation. + + [`system` configuration option]: @docroot@/command-ref/conf-file.md#conf-system + +- [`builder`]{#attr-builder} ([Path](@docroot@/language/values.md#type-path) | [String](@docroot@/language/values.md#type-string)) + + Path to an executable that will perform the build. + + Examples: + + `builder = "/bin/bash";` + + `builder = ./builder.sh;` + + `builder = "${pkgs.python}/bin/python";` + +### Optional + +- [`args`]{#attr-args} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) Default: `[ ]` + + Command-line arguments to be passed to the [`builder`](#attr-builder) executable. + + Example: `args = [ "-c" "echo hello world > $out" ];` + +- [`outputs`]{#attr-outputs} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) Default: `[ "out" ]` + + Symbolic outputs of the derivation. + Each output name is passed to the [`builder`](#attr-builder) executable as an environment variable with its value set to the corresponding [output path]. + + [output path]: @docroot@/glossary.md#gloss-output-path + + By default, a derivation produces a single output path called `out`. + However, derivations can produce multiple output paths. + This allows the associated [store objects](@docroot@/glossary.md#gloss-store-object) and their [closures](@docroot@/glossary.md#gloss-closure) to be copied or garbage-collected separately. + + Examples: + + Imagine a library package that provides a dynamic library, header files, and documentation. + A program that links against the library doesn’t need the header files and documentation at runtime, and it doesn’t need the documentation at build time. + Thus, the library package could specify: + + ```nix + outputs = [ "lib" "headers" "doc" ]; + ``` + + This will cause Nix to pass environment variables `lib`, `headers`, and `doc` to the builder containing the intended store paths of each output. + The builder would typically do something like + + ```bash + ./configure \ + --libdir=$lib/lib \ + --includedir=$headers/include \ + --docdir=$doc/share/doc + ``` + + for an Autoconf-style package. + + You can refer to each output of a + derivation by selecting it as an attribute, e.g. + + ```nix + buildInputs = [ pkg.lib pkg.headers ]; + ``` + + + The first element of `outputs` determines the *default output*. + Thus, you could also write + + ```nix + buildInputs = [ pkg pkg.headers ]; + ``` + + since `pkg` is equivalent to `pkg.lib`. + +- See [Advanced Attributes](./advanced-attributes.md) for more, infrequently used, optional attributes. + + + +- Every other attribute is passed as an environment variable to the builder. + Attribute values are translated to environment variables as follows: + + - Strings and numbers are just passed verbatim. + + - A *path* (e.g., `../foo/sources.tar`) causes the referenced file + to be copied to the store; its location in the store is put in + the environment variable. The idea is that all sources should + reside in the Nix store, since all inputs to a derivation should + reside in the Nix store. + + - A *derivation* causes that derivation to be built prior to the + present derivation; its default output path is put in the + environment variable. + + - Lists of the previous types are also allowed. They are simply + concatenated, separated by spaces. + + - `true` is passed as the string `1`, `false` and `null` are + passed as an empty string. ## Builder execution From d621dd17f23023d4d6a84daba90ef38e848a1375 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 15 Sep 2023 10:31:31 +0200 Subject: [PATCH 123/139] more precise wording Co-authored-by: Robert Hensing --- doc/manual/src/language/derivations.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index 56c6818ca..dfefe446f 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -14,14 +14,14 @@ It outputs an attribute set, and produces a [store derivation](@docroot@/glossar - [`name`]{#attr-name} ([String](@docroot@/language/values.md#type-string)) - Symbolic name for the derivation. - It is appended to the [store paths](@docroot@/glossary.md#gloss-store-path) of the resulting store derivation and its [output paths][output path]. + A symbolic name for the derivation. + It is added to the [store derivation]'s [path](@docroot@/glossary.md#gloss-store-path) and its [output paths][output path]. Example: `name = "hello";` - [`system`]{#attr-system} ([String](@docroot@/language/values.md#type-string)) - The system type on which the [`builder`](#attr-builder) executable can be run. + The system type on which the [`builder`](#attr-builder) executable is meant to be run. Nix will only build derivations where the `system` attribute matches the current [`system` configuration option]. It can automatically [build on other platforms](../advanced-topics/distributed-builds.md) by forwarding build requests to other machines. From 026c24e3787d7b1cf81c49fdcb66818fd5dad2e4 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 15 Sep 2023 15:59:07 +0200 Subject: [PATCH 124/139] add example for store path using the given name --- doc/manual/src/language/derivations.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index dfefe446f..3ced976f6 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -19,6 +19,7 @@ It outputs an attribute set, and produces a [store derivation](@docroot@/glossar Example: `name = "hello";` + The store derivation's path will be `/nix/store/-hello.drv`, and the output paths will be of the form `/nix/store/-hello[-]` - [`system`]{#attr-system} ([String](@docroot@/language/values.md#type-string)) The system type on which the [`builder`](#attr-builder) executable is meant to be run. From 17884f54d18b6d69dbfefe6c08272ea51b8879f8 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Fri, 22 Sep 2023 09:53:29 +0200 Subject: [PATCH 125/139] clarification on extra attributes Co-authored-by: Robert Hensing --- doc/manual/src/language/derivations.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index 3ced976f6..77eac03b6 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -114,7 +114,11 @@ It outputs an attribute set, and produces a [store derivation](@docroot@/glossar - Every other attribute is passed as an environment variable to the builder. Attribute values are translated to environment variables as follows: - - Strings and numbers are just passed verbatim. + - Strings are passed unchanged. + + - Integral numbers are converted to decimal notation. + + - Floating point numbers are converted to simple decimal or scientific notation with a preset precision. - A *path* (e.g., `../foo/sources.tar`) causes the referenced file to be copied to the store; its location in the store is put in From 5e4734a08b8fb3f80b9617c190a2b5dfaba4a1da Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 26 Sep 2023 00:13:56 +0200 Subject: [PATCH 126/139] reword to avoid saying "build" derivations are about data transformation, so the term "build" does not add any information. there was also some feedback that "build task" is not more helpful than "derivation" if you have no prior experience with Nix or build systems, while existing associations may be misleading. --- doc/manual/src/language/derivations.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index 77eac03b6..da3cca4ad 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -1,9 +1,9 @@ # Derivations The most important built-in function is `derivation`, which is used to describe a single derivation: -a build task to run a process on precisely defined input files and repeatably produce output files at uniquely determined file system paths. +a specification for running an executable on precisely defined input files to repeatably produce output files at uniquely determined file system paths. -It takes as input an attribute set, the attributes of which specify the inputs to the build. +It takes as input an attribute set, the attributes of which specify the inputs to the process. It outputs an attribute set, and produces a [store derivation](@docroot@/glossary.md#gloss-store-derivation) as a side effect of evaluation. From 75a231147f2d6acd32321e088a5030a69f188854 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 26 Sep 2023 00:43:41 +0200 Subject: [PATCH 127/139] be more precise about `system` semantics --- doc/manual/src/language/derivations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index da3cca4ad..c13c81e7d 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -24,7 +24,7 @@ It outputs an attribute set, and produces a [store derivation](@docroot@/glossar The system type on which the [`builder`](#attr-builder) executable is meant to be run. - Nix will only build derivations where the `system` attribute matches the current [`system` configuration option]. + A necessary condition for Nix to build derivations locally is that the `system` attribute matches the current [`system` configuration option]. It can automatically [build on other platforms](../advanced-topics/distributed-builds.md) by forwarding build requests to other machines. Examples: From 7de66f19f84edc0d617e00f6e0cc37f2dbc4f5f6 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 26 Sep 2023 00:46:26 +0200 Subject: [PATCH 128/139] example: headers -> dev make the example more realistic, since `headers` is not an output name used in Nixpkgs Co-authored-by: Robert Hensing --- doc/manual/src/language/derivations.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index c13c81e7d..6e6d3298b 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -75,16 +75,16 @@ It outputs an attribute set, and produces a [store derivation](@docroot@/glossar Thus, the library package could specify: ```nix - outputs = [ "lib" "headers" "doc" ]; + outputs = [ "lib" "dev" "doc" ]; ``` - This will cause Nix to pass environment variables `lib`, `headers`, and `doc` to the builder containing the intended store paths of each output. + This will cause Nix to pass environment variables `lib`, `dev`, and `doc` to the builder containing the intended store paths of each output. The builder would typically do something like ```bash ./configure \ --libdir=$lib/lib \ - --includedir=$headers/include \ + --includedir=$dev/include \ --docdir=$doc/share/doc ``` @@ -94,7 +94,7 @@ It outputs an attribute set, and produces a [store derivation](@docroot@/glossar derivation by selecting it as an attribute, e.g. ```nix - buildInputs = [ pkg.lib pkg.headers ]; + buildInputs = [ pkg.lib pkg.dev ]; ``` @@ -102,7 +102,7 @@ It outputs an attribute set, and produces a [store derivation](@docroot@/glossar Thus, you could also write ```nix - buildInputs = [ pkg pkg.headers ]; + buildInputs = [ pkg pkg.dev ]; ``` since `pkg` is equivalent to `pkg.lib`. From 5b0336b3b14551762f37126423ea8b4590ebb560 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 26 Sep 2023 00:58:50 +0200 Subject: [PATCH 129/139] reword example for clarity --- doc/manual/src/language/derivations.md | 28 ++++++++++---------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index 6e6d3298b..e0b9d021a 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -71,11 +71,15 @@ It outputs an attribute set, and produces a [store derivation](@docroot@/glossar Examples: Imagine a library package that provides a dynamic library, header files, and documentation. - A program that links against the library doesn’t need the header files and documentation at runtime, and it doesn’t need the documentation at build time. + A program that links against such a library doesn’t need the header files and documentation at runtime, and it doesn’t need the documentation at build time. Thus, the library package could specify: ```nix - outputs = [ "lib" "dev" "doc" ]; + derivation { + # ... + outputs = [ "lib" "dev" "doc" ]; + # ... + } ``` This will cause Nix to pass environment variables `lib`, `dev`, and `doc` to the builder containing the intended store paths of each output. @@ -90,22 +94,12 @@ It outputs an attribute set, and produces a [store derivation](@docroot@/glossar for an Autoconf-style package. - You can refer to each output of a - derivation by selecting it as an attribute, e.g. - - ```nix - buildInputs = [ pkg.lib pkg.dev ]; - ``` - + You can refer to each output of a derivation by selecting it as an attribute, e.g. `myPackage.lib` or `myPackage.doc`. The first element of `outputs` determines the *default output*. - Thus, you could also write + Therefore, in the given example, `myPackage` is equivalent to `myPackage.lib`. - ```nix - buildInputs = [ pkg pkg.dev ]; - ``` - - since `pkg` is equivalent to `pkg.lib`. + - See [Advanced Attributes](./advanced-attributes.md) for more, infrequently used, optional attributes. @@ -115,9 +109,9 @@ It outputs an attribute set, and produces a [store derivation](@docroot@/glossar Attribute values are translated to environment variables as follows: - Strings are passed unchanged. - + - Integral numbers are converted to decimal notation. - + - Floating point numbers are converted to simple decimal or scientific notation with a preset precision. - A *path* (e.g., `../foo/sources.tar`) causes the referenced file From 45de35bcf14d1bdad5dca2e24a113d26d1985aed Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 26 Sep 2023 01:24:07 +0200 Subject: [PATCH 130/139] fix broken redirects script --- doc/manual/redirects.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index b43622ed6..b2fad19bb 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -343,7 +343,7 @@ const redirects = { "linux": "uninstall.html#linux", "macos": "uninstall.html#macos", "uninstalling": "uninstall.html", - } + }, "contributing/hacking.html": { "nix-with-flakes": "#building-nix-with-flakes", "classic-nix": "#building-nix", From c6f8247032612b4cc4a9f79213a887aa9ddddfda Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 26 Sep 2023 03:10:12 +0200 Subject: [PATCH 131/139] fix broken reference link --- doc/manual/src/language/derivations.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index e0b9d021a..4a1b70fd1 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -4,7 +4,9 @@ The most important built-in function is `derivation`, which is used to describe a specification for running an executable on precisely defined input files to repeatably produce output files at uniquely determined file system paths. It takes as input an attribute set, the attributes of which specify the inputs to the process. -It outputs an attribute set, and produces a [store derivation](@docroot@/glossary.md#gloss-store-derivation) as a side effect of evaluation. +It outputs an attribute set, and produces a [store derivation] as a side effect of evaluation. + +[store derivation]: @docroot@/glossary.md#gloss-store-derivation From 7e24dc606b1cf7fd4e4dfd7027ba1469a59784d5 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Fri, 22 Sep 2023 13:05:55 -0400 Subject: [PATCH 132/139] fix(tests): fix assumption that string.s is a char* --- src/libexpr/tests/libexpr.hh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libexpr/tests/libexpr.hh b/src/libexpr/tests/libexpr.hh index b8e65aafe..97b05ac46 100644 --- a/src/libexpr/tests/libexpr.hh +++ b/src/libexpr/tests/libexpr.hh @@ -71,7 +71,7 @@ namespace nix { if (arg.type() != nString) { return false; } - return std::string_view(arg.string.s) == s; + return std::string_view(arg.c_str()) == s; } MATCHER_P(IsIntEq, v, fmt("The string is equal to \"%1%\"", v)) { @@ -106,8 +106,8 @@ namespace nix { if (arg.type() != nPath) { *result_listener << "Expected a path got " << arg.type(); return false; - } else if (std::string_view(arg.string.s) != p) { - *result_listener << "Expected a path that equals \"" << p << "\" but got: " << arg.string.s; + } else if (std::string_view(arg._path) != p) { + *result_listener << "Expected a path that equals \"" << p << "\" but got: " << arg.c_str(); return false; } return true; From b17f200b11b3901c4f975c9901f1e2c6fc54124f Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 26 Sep 2023 03:49:03 +0200 Subject: [PATCH 133/139] Document "Import From Derivation" (#7332) * document "Import From Derivation" Co-authored-by: Robert Hensing Co-authored-by: John Ericson --- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/glossary.md | 5 +- .../src/language/import-from-derivation.md | 139 ++++++++++++++++++ src/libexpr/eval-settings.hh | 11 +- 4 files changed, 150 insertions(+), 6 deletions(-) create mode 100644 doc/manual/src/language/import-from-derivation.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index c23186511..9e2ad0618 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -33,6 +33,7 @@ - [Operators](language/operators.md) - [Derivations](language/derivations.md) - [Advanced Attributes](language/advanced-attributes.md) + - [Import From Derivation](language/import-from-derivation.md) - [Built-in Constants](language/builtin-constants.md) - [Built-in Functions](language/builtins.md) - [Advanced Topics](advanced-topics/advanced-topics.md) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index b7c340d36..c23466351 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -105,12 +105,15 @@ - [store object]{#gloss-store-object} - A store object consists of a [file system object], [reference]s to other store objects, and other metadata. It can be referred to by a [store path]. [store object]: #gloss-store-object +- [IFD]{#gloss-ifd} + + [Import From Derivation](./language/import-from-derivation.md) + - [input-addressed store object]{#gloss-input-addressed-store-object} A store object produced by building a diff --git a/doc/manual/src/language/import-from-derivation.md b/doc/manual/src/language/import-from-derivation.md new file mode 100644 index 000000000..03b3f9d91 --- /dev/null +++ b/doc/manual/src/language/import-from-derivation.md @@ -0,0 +1,139 @@ +# Import From Derivation + +The value of a Nix expression can depend on the contents of a [store object](@docroot@/glossary.md#gloss-store-object). + +Passing an expression `expr` that evaluates to a [store path](@docroot@/glossary.md#gloss-store-path) to any built-in function which reads from the filesystem constitutes Import From Derivation (IFD): + +- [`import`](./builtins.md#builtins-import)` expr` +- [`builtins.readFile`](./builtins.md#builtins-readFile)` expr` +- [`builtins.readFileType`](./builtins.md#builtins-readFileType)` expr` +- [`builtins.readDir`](./builtins.md#builtins-readDir)` expr` +- [`builtins.pathExists`](./builtins.md#builtins-pathExists)` expr` +- [`builtins.filterSource`](./builtins.md#builtins-filterSource)` f expr` +- [`builtins.path`](./builtins.md#builtins-path)` { path = expr; }` +- [`builtins.hashFile`](./builtins.md#builtins-hashFile)` t expr` +- `builtins.scopedImport x drv` + +When the store path needs to be accessed, evaluation will be paused, the corresponding store object [realised], and then evaluation resumed. + +[realised]: @docroot@/glossary.md#gloss-realise + +This has performance implications: +Evaluation can only finish when all required store objects are realised. +Since the Nix language evaluator is sequential, it only finds store paths to read from one at a time. +While realisation is always parallel, in this case it cannot be done for all required store paths at once, and is therefore much slower than otherwise. + +Realising store objects during evaluation can be disabled by setting [`allow-import-from-derivation`](../command-ref/conf-file.md#conf-allow-import-from-derivation) to `false`. +Without IFD it is ensured that evaluation is complete and Nix can produce a build plan before starting any realisation. + +## Example + +In the following Nix expression, the inner derivation `drv` produces a file with contents `hello`. + +```nix +# IFD.nix +let + drv = derivation { + name = "hello"; + builder = "/bin/sh"; + args = [ "-c" "echo -n hello > $out" ]; + system = builtins.currentSystem; + }; +in "${builtins.readFile drv} world" +``` + +```shellSession +nix-instantiate IFD.nix --eval --read-write-mode +``` + +``` +building '/nix/store/348q1cal6sdgfxs8zqi9v8llrsn4kqkq-hello.drv'... +"hello world" +``` + +The contents of the derivation's output have to be [realised] before they can be read with [`readFile`](./builtins.md#builtins-readFile). +Only then evaluation can continue to produce the final result. + +## Illustration + +As a first approximation, the following data flow graph shows how evaluation and building are interleaved, if the value of a Nix expression depends on realising a [store object]. +Boxes are data structures, arrow labels are transformations. + +``` ++----------------------+ +------------------------+ +| Nix evaluator | | Nix store | +| .----------------. | | | +| | Nix expression | | | | +| '----------------' | | | +| | | | | +| evaluate | | | +| | | | | +| V | | | +| .------------. | | .------------------. | +| | derivation |----|-instantiate-|->| store derivation | | +| '------------' | | '------------------' | +| | | | | +| | | realise | +| | | | | +| | | V | +| .----------------. | | .--------------. | +| | Nix expression |<-|----read-----|----| store object | | +| '----------------' | | '--------------' | +| | | | | +| evaluate | | | +| | | | | +| V | | | +| .------------. | | | +| | value | | | | +| '------------' | | | ++----------------------+ +------------------------+ +``` + +In more detail, the following sequence diagram shows how the expression is evaluated step by step, and where evaluation is blocked to wait for the build output to appear. + +``` +.-------. .-------------. .---------. +|Nix CLI| |Nix evaluator| |Nix store| +'-------' '-------------' '---------' + | | | + |evaluate IFD.nix| | + |--------------->| | + | | | + | evaluate `"${readFile drv} world"` | + | | | + | evaluate `readFile drv` | + | | | + | evaluate `drv` as string | + | | | + | |instantiate /nix/store/...-hello.drv| + | |----------------------------------->| + | : | + | : realise /nix/store/...-hello.drv | + | :----------------------------------->| + | : | + | |--------. + | : | | + | (evaluation blocked) | echo hello > $out + | : | | + | |<-------' + | : /nix/store/...-hello | + | |<-----------------------------------| + | | | + | resume `readFile /nix/store/...-hello` | + | | | + | | readFile /nix/store/...-hello | + | |----------------------------------->| + | | | + | | hello | + | |<-----------------------------------| + | | | + | resume `"${"hello"} world"` | + | | | + | resume `"hello world"` | + | | | + | "hello world" | | + |<---------------| | +.-------. .-------------. .---------. +|Nix CLI| |Nix evaluator| |Nix store| +'-------' '-------------' '---------' +``` diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index 19122bc31..19ba679be 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -47,11 +47,12 @@ struct EvalSettings : Config Setting enableImportFromDerivation{ this, true, "allow-import-from-derivation", R"( - By default, Nix allows you to `import` from a derivation, allowing - building at evaluation time. With this option set to false, Nix will - throw an error when evaluating an expression that uses this feature, - allowing users to ensure their evaluation will not require any - builds to take place. + By default, Nix allows [Import from Derivation](@docroot@/language/import-from-derivation.md). + + With this option set to `false`, Nix will throw an error when evaluating an expression that uses this feature, + even when the required store object is readily available. + This ensures that evaluation will not require any builds to take place, + regardless of the state of the store. )"}; Setting allowedUris{this, {}, "allowed-uris", From 5b902ce9d60fca33c0f5bb76fcd31792c0bb03a6 Mon Sep 17 00:00:00 2001 From: Yingchi Long Date: Tue, 26 Sep 2023 23:30:32 +0800 Subject: [PATCH 134/139] libexpr: construct ExprPath by move ctor, not copy cotr --- src/libexpr/parser.y | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 792f51fde..509f4fcc4 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -520,7 +520,7 @@ path_start /* add back in the trailing '/' to the first segment */ if ($1.p[$1.l-1] == '/' && $1.l > 1) path += "/"; - $$ = new ExprPath(path); + $$ = new ExprPath(std::move(path)); } | HPATH { if (evalSettings.pureEval) { @@ -530,7 +530,7 @@ path_start ); } Path path(getHome() + std::string($1.p + 1, $1.l - 1)); - $$ = new ExprPath(path); + $$ = new ExprPath(std::move(path)); } ; From 1eeea01931d20d32c917e92d860513b7cb32ba10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christina=20S=C3=B8rensen?= Date: Wed, 27 Sep 2023 02:07:43 +0000 Subject: [PATCH 135/139] Fix repl.md duplicate typo Seems like `legacyPackages.x86_64-linux.emacs.name` is accidentally shown twice. --- src/nix/repl.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/nix/repl.md b/src/nix/repl.md index c5113be61..9dc61fd35 100644 --- a/src/nix/repl.md +++ b/src/nix/repl.md @@ -43,9 +43,6 @@ R""( nix-repl> legacyPackages.x86_64-linux.emacs.name "emacs-27.1" - nix-repl> legacyPackages.x86_64-linux.emacs.name - "emacs-27.1" - nix-repl> :q # nix repl --expr 'import {}' From 399ef8442012308f480ea5841fb6ac732207e03d Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Mon, 25 Sep 2023 21:30:41 -0400 Subject: [PATCH 136/139] refactor: use string accessors Create context, string_view, and c_str, accessors throughout in order to better support improvements to the underlying string representation. --- src/libcmd/repl.cc | 2 +- src/libexpr/eval-cache.cc | 8 ++++---- src/libexpr/eval.cc | 18 +++++++++--------- src/libexpr/flake/flake.cc | 10 +++++----- src/libexpr/get-drvs.cc | 12 ++++++------ src/libexpr/primops.cc | 10 +++++----- src/libexpr/primops/fetchClosure.cc | 2 +- src/libexpr/tests/primops.cc | 4 ++-- src/libexpr/value-to-json.cc | 2 +- src/libexpr/value-to-xml.cc | 6 +++--- src/libexpr/value.hh | 20 +++++++++++++++----- src/nix-env/nix-env.cc | 6 +++--- src/nix/eval.cc | 2 +- 13 files changed, 56 insertions(+), 46 deletions(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 944609f22..2e17a29a7 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -922,7 +922,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m case nString: str << ANSI_WARNING; - printLiteralString(str, v.string.s); + printLiteralString(str, v.string_view()); str << ANSI_NORMAL; break; diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 2c9aa5532..391b32a77 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -440,8 +440,8 @@ Value & AttrCursor::forceValue() if (root->db && (!cachedValue || std::get_if(&cachedValue->second))) { if (v.type() == nString) - cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context), - string_t{v.string.s, {}}}; + cachedValue = {root->db->setString(getKey(), v.c_str(), v.context()), + string_t{v.c_str(), {}}}; else if (v.type() == nPath) { auto path = v.path().path; cachedValue = {root->db->setString(getKey(), path.abs()), string_t{path.abs(), {}}}; @@ -582,7 +582,7 @@ std::string AttrCursor::getString() if (v.type() != nString && v.type() != nPath) root->state.error("'%s' is not a string but %s", getAttrPathStr()).debugThrow(); - return v.type() == nString ? v.string.s : v.path().to_string(); + return v.type() == nString ? v.c_str() : v.path().to_string(); } string_t AttrCursor::getStringWithContext() @@ -624,7 +624,7 @@ string_t AttrCursor::getStringWithContext() if (v.type() == nString) { NixStringContext context; copyContext(v, context); - return {v.string.s, std::move(context)}; + return {v.c_str(), std::move(context)}; } else if (v.type() == nPath) return {v.path().to_string(), {}}; diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index afa864730..a78c0afb1 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -114,7 +114,7 @@ void Value::print(const SymbolTable &symbols, std::ostream &str, printLiteralBool(str, boolean); break; case tString: - printLiteralString(str, string.s); + printLiteralString(str, string_view()); break; case tPath: str << path().to_string(); // !!! escaping? @@ -339,7 +339,7 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env) Value nameValue; name.expr->eval(state, env, nameValue); state.forceStringNoCtx(nameValue, noPos, "while evaluating an attribute name"); - return state.symbols.create(nameValue.string.s); + return state.symbols.create(nameValue.string_view()); } } @@ -1343,7 +1343,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) if (nameVal.type() == nNull) continue; state.forceStringNoCtx(nameVal, i.pos, "while evaluating the name of a dynamic attribute"); - auto nameSym = state.symbols.create(nameVal.string.s); + auto nameSym = state.symbols.create(nameVal.string_view()); Bindings::iterator j = v.attrs->find(nameSym); if (j != v.attrs->end()) state.error("dynamic attribute '%1%' already defined at %2%", state.symbols[nameSym], state.positions[j->pos]).atPos(i.pos).withFrame(env, *this).debugThrow(); @@ -2155,7 +2155,7 @@ std::string_view EvalState::forceString(Value & v, const PosIdx pos, std::string forceValue(v, pos); if (v.type() != nString) error("value is %1% while a string was expected", showType(v)).debugThrow(); - return v.string.s; + return v.string_view(); } catch (Error & e) { e.addTrace(positions[pos], errorCtx); throw; @@ -2182,8 +2182,8 @@ std::string_view EvalState::forceString(Value & v, NixStringContext & context, c std::string_view EvalState::forceStringNoCtx(Value & v, const PosIdx pos, std::string_view errorCtx) { auto s = forceString(v, pos, errorCtx); - if (v.string.context) { - error("the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string.s, v.string.context[0]).withTrace(pos, errorCtx).debugThrow(); + if (v.context()) { + error("the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string_view(), v.context()[0]).withTrace(pos, errorCtx).debugThrow(); } return s; } @@ -2196,7 +2196,7 @@ bool EvalState::isDerivation(Value & v) if (i == v.attrs->end()) return false; forceValue(*i->value, i->pos); if (i->value->type() != nString) return false; - return strcmp(i->value->string.s, "derivation") == 0; + return i->value->string_view().compare("derivation") == 0; } @@ -2228,7 +2228,7 @@ BackedStringView EvalState::coerceToString( if (v.type() == nString) { copyContext(v, context); - return std::string_view(v.string.s); + return v.string_view(); } if (v.type() == nPath) { @@ -2426,7 +2426,7 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v return v1.boolean == v2.boolean; case nString: - return strcmp(v1.string.s, v2.string.s) == 0; + return v1.string_view().compare(v2.string_view()) == 0; case nPath: return strcmp(v1._path, v2._path) == 0; diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 90427e064..a6212c12f 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -113,7 +113,7 @@ static FlakeInput parseFlakeInput(EvalState & state, try { if (attr.name == sUrl) { expectType(state, nString, *attr.value, attr.pos); - url = attr.value->string.s; + url = attr.value->string_view(); attrs.emplace("url", *url); } else if (attr.name == sFlake) { expectType(state, nBool, *attr.value, attr.pos); @@ -122,7 +122,7 @@ static FlakeInput parseFlakeInput(EvalState & state, input.overrides = parseFlakeInputs(state, attr.value, attr.pos, baseDir, lockRootPath); } else if (attr.name == sFollows) { expectType(state, nString, *attr.value, attr.pos); - auto follows(parseInputPath(attr.value->string.s)); + auto follows(parseInputPath(attr.value->c_str())); follows.insert(follows.begin(), lockRootPath.begin(), lockRootPath.end()); input.follows = follows; } else { @@ -131,7 +131,7 @@ static FlakeInput parseFlakeInput(EvalState & state, #pragma GCC diagnostic ignored "-Wswitch-enum" switch (attr.value->type()) { case nString: - attrs.emplace(state.symbols[attr.name], attr.value->string.s); + attrs.emplace(state.symbols[attr.name], attr.value->c_str()); break; case nBool: attrs.emplace(state.symbols[attr.name], Explicit { attr.value->boolean }); @@ -229,7 +229,7 @@ static Flake getFlake( if (auto description = vInfo.attrs->get(state.sDescription)) { expectType(state, nString, *description->value, description->pos); - flake.description = description->value->string.s; + flake.description = description->value->c_str(); } auto sInputs = state.symbols.create("inputs"); @@ -850,7 +850,7 @@ static void prim_flakeRefToString( Explicit { attr.value->boolean }); } else if (t == nString) { attrs.emplace(state.symbols[attr.name], - std::string(attr.value->str())); + std::string(attr.value->string_view())); } else { state.error( "flake reference attribute sets may only contain integers, Booleans, " diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 506a63677..fe3e6f7ee 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -156,7 +156,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall Outputs result; for (auto elem : outTI->listItems()) { if (elem->type() != nString) throw errMsg; - auto out = outputs.find(elem->string.s); + auto out = outputs.find(elem->c_str()); if (out == outputs.end()) throw errMsg; result.insert(*out); } @@ -230,7 +230,7 @@ std::string DrvInfo::queryMetaString(const std::string & name) { Value * v = queryMeta(name); if (!v || v->type() != nString) return ""; - return v->string.s; + return v->c_str(); } @@ -242,7 +242,7 @@ NixInt DrvInfo::queryMetaInt(const std::string & name, NixInt def) if (v->type() == nString) { /* Backwards compatibility with before we had support for integer meta fields. */ - if (auto n = string2Int(v->string.s)) + if (auto n = string2Int(v->c_str())) return *n; } return def; @@ -256,7 +256,7 @@ NixFloat DrvInfo::queryMetaFloat(const std::string & name, NixFloat def) if (v->type() == nString) { /* Backwards compatibility with before we had support for float meta fields. */ - if (auto n = string2Float(v->string.s)) + if (auto n = string2Float(v->c_str())) return *n; } return def; @@ -271,8 +271,8 @@ bool DrvInfo::queryMetaBool(const std::string & name, bool def) if (v->type() == nString) { /* Backwards compatibility with before we had support for Boolean meta fields. */ - if (strcmp(v->string.s, "true") == 0) return true; - if (strcmp(v->string.s, "false") == 0) return false; + if (v->string_view() == "true") return true; + if (v->string_view() == "false") return false; } return def; } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index cd9a05bb2..1cb5cdf30 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -590,7 +590,7 @@ struct CompareValues case nFloat: return v1->fpoint < v2->fpoint; case nString: - return strcmp(v1->string.s, v2->string.s) < 0; + return v1->string_view().compare(v2->string_view()) < 0; case nPath: return strcmp(v1->_path, v2->_path) < 0; case nList: @@ -982,7 +982,7 @@ static void prim_trace(EvalState & state, const PosIdx pos, Value * * args, Valu { state.forceValue(*args[0], pos); if (args[0]->type() == nString) - printError("trace: %1%", args[0]->string.s); + printError("trace: %1%", args[0]->string_view()); else printError("trace: %1%", printValue(state, *args[0])); state.forceValue(*args[1], pos); @@ -1528,7 +1528,7 @@ static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args, auto path = realisePath(state, pos, arg, { .checkForPureEval = false }); /* SourcePath doesn't know about trailing slash. */ - auto mustBeDir = arg.type() == nString && arg.str().ends_with("/"); + auto mustBeDir = arg.type() == nString && arg.string_view().ends_with("/"); try { auto checked = state.checkSourcePath(path); @@ -2400,7 +2400,7 @@ static void prim_attrNames(EvalState & state, const PosIdx pos, Value * * args, (v.listElems()[n++] = state.allocValue())->mkString(state.symbols[i.name]); std::sort(v.listElems(), v.listElems() + n, - [](Value * v1, Value * v2) { return strcmp(v1->string.s, v2->string.s) < 0; }); + [](Value * v1, Value * v2) { return v1->string_view().compare(v2->string_view()) < 0; }); } static RegisterPrimOp primop_attrNames({ @@ -2541,7 +2541,7 @@ static void prim_removeAttrs(EvalState & state, const PosIdx pos, Value * * args names.reserve(args[1]->listSize()); for (auto elem : args[1]->listItems()) { state.forceStringNoCtx(*elem, pos, "while evaluating the values of the second argument passed to builtins.removeAttrs"); - names.emplace_back(state.symbols.create(elem->string.s), nullptr); + names.emplace_back(state.symbols.create(elem->string_view()), nullptr); } std::sort(names.begin(), names.end()); diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 7fe8203f4..b86ef6b93 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -133,7 +133,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg else if (attrName == "toPath") { state.forceValue(*attr.value, attr.pos); - bool isEmptyString = attr.value->type() == nString && attr.value->string.s == std::string(""); + bool isEmptyString = attr.value->type() == nString && attr.value->string_view() == ""; if (isEmptyString) { toPath = StorePathOrGap {}; } diff --git a/src/libexpr/tests/primops.cc b/src/libexpr/tests/primops.cc index ce3b5d11f..d820b860e 100644 --- a/src/libexpr/tests/primops.cc +++ b/src/libexpr/tests/primops.cc @@ -711,14 +711,14 @@ namespace nix { // FIXME: add a test that verifies the string context is as expected auto v = eval("builtins.replaceStrings [\"oo\" \"a\"] [\"a\" \"i\"] \"foobar\""); ASSERT_EQ(v.type(), nString); - ASSERT_EQ(v.string.s, std::string_view("fabir")); + ASSERT_EQ(v.string_view(), "fabir"); } TEST_F(PrimOpTest, concatStringsSep) { // FIXME: add a test that verifies the string context is as expected auto v = eval("builtins.concatStringsSep \"%\" [\"foo\" \"bar\" \"baz\"]"); ASSERT_EQ(v.type(), nString); - ASSERT_EQ(std::string_view(v.string.s), "foo%bar%baz"); + ASSERT_EQ(v.string_view(), "foo%bar%baz"); } TEST_F(PrimOpTest, split1) { diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index ac3986c87..cbc91f509 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -31,7 +31,7 @@ json printValueAsJSON(EvalState & state, bool strict, case nString: copyContext(v, context); - out = v.string.s; + out = v.c_str(); break; case nPath: diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 2539ad1c1..bd7a4ae30 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -74,7 +74,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, case nString: /* !!! show the context? */ copyContext(v, context); - doc.writeEmptyElement("string", singletonAttrs("value", v.string.s)); + doc.writeEmptyElement("string", singletonAttrs("value", v.c_str())); break; case nPath: @@ -96,14 +96,14 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, if (a != v.attrs->end()) { if (strict) state.forceValue(*a->value, a->pos); if (a->value->type() == nString) - xmlAttrs["drvPath"] = drvPath = a->value->string.s; + xmlAttrs["drvPath"] = drvPath = a->value->c_str(); } a = v.attrs->find(state.sOutPath); if (a != v.attrs->end()) { if (strict) state.forceValue(*a->value, a->pos); if (a->value->type() == nString) - xmlAttrs["outPath"] = a->value->string.s; + xmlAttrs["outPath"] = a->value->c_str(); } XMLOpenElement _(doc, "derivation", xmlAttrs); diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index c44683e50..7a1165cac 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -186,10 +186,9 @@ public: * For canonicity, the store paths should be in sorted order. */ struct { - const char * s; + const char * c_str; const char * * context; // must be in sorted order } string; - const char * _path; Bindings * attrs; struct { @@ -270,7 +269,7 @@ public: inline void mkString(const char * s, const char * * context = 0) { internalType = tString; - string.s = s; + string.c_str = s; string.context = context; } @@ -441,10 +440,21 @@ public: return SourcePath{CanonPath(_path)}; } - std::string_view str() const + std::string_view string_view() const { assert(internalType == tString); - return std::string_view(string.s); + return std::string_view(string.c_str); + } + + const char * const c_str() const + { + assert(internalType == tString); + return string.c_str; + } + + const char * * context() const + { + return string.context; } }; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index b112e8cb3..e455a50fd 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1228,7 +1228,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) else { if (v->type() == nString) { attrs2["type"] = "string"; - attrs2["value"] = v->string.s; + attrs2["value"] = v->c_str(); xml.writeEmptyElement("meta", attrs2); } else if (v->type() == nInt) { attrs2["type"] = "int"; @@ -1248,7 +1248,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) for (auto elem : v->listItems()) { if (elem->type() != nString) continue; XMLAttrs attrs3; - attrs3["value"] = elem->string.s; + attrs3["value"] = elem->c_str(); xml.writeEmptyElement("string", attrs3); } } else if (v->type() == nAttrs) { @@ -1260,7 +1260,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) if(a.value->type() != nString) continue; XMLAttrs attrs3; attrs3["type"] = globals.state->symbols[i.name]; - attrs3["value"] = a.value->string.s; + attrs3["value"] = a.value->c_str(); xml.writeEmptyElement("string", attrs3); } } diff --git a/src/nix/eval.cc b/src/nix/eval.cc index d880bef0a..b34af34e0 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -85,7 +85,7 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption state->forceValue(v, pos); if (v.type() == nString) // FIXME: disallow strings with contexts? - writeFile(path, v.string.s); + writeFile(path, v.string_view()); else if (v.type() == nAttrs) { if (mkdir(path.c_str(), 0777) == -1) throw SysError("creating directory '%s'", path); From 5bc540a8ca7ada282839ab58225a1a24cc723c4e Mon Sep 17 00:00:00 2001 From: Alex Ameen Date: Wed, 27 Sep 2023 14:49:52 -0500 Subject: [PATCH 137/139] Respect `NOCOLOR` While `nix` has always been respectful towards requests for `NO_COLOR=1`, this change asks represents a new stage of maturity for `nix` - making it also respect quests for `NOCOLOR=1`. This ideally makes the tool more accessible to folks like me, who are exhausted by guessing whether `NO_COLOR` or `NOCOLOR` is the right environment variable to set. <3 --- src/libutil/util.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 5a10c69e2..3b4c181e5 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1515,7 +1515,7 @@ bool shouldANSI() { return isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb" - && !getEnv("NO_COLOR").has_value(); + && !(getEnv("NO_COLOR").has_value() || getEnv("NOCOLOR").has_value()); } std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int width) From dafa38213bda956525fb692eea33f2ec4fed7ff3 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 27 Sep 2023 23:10:39 +0100 Subject: [PATCH 138/139] Update doc/manual/src/advanced-topics/post-build-hook.md Co-authored-by: Valentin Gagarin --- doc/manual/src/advanced-topics/post-build-hook.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/advanced-topics/post-build-hook.md b/doc/manual/src/advanced-topics/post-build-hook.md index 17b67111c..f7b3b1317 100644 --- a/doc/manual/src/advanced-topics/post-build-hook.md +++ b/doc/manual/src/advanced-topics/post-build-hook.md @@ -69,7 +69,7 @@ exec nix copy --to "s3://example-nix-cache" $OUT_PATHS > store sign`. Nix guarantees the paths will not contain any spaces, > however a store path might contain glob characters. The `set -f` > disables globbing in the shell. -> If you want to upload the .drv file too, the `$DRV_PATH` variable +> If you want to upload the `.drv` file too, the `$DRV_PATH` variable > is also defined for the script and works just like `$OUT_PATHS`. Then make sure the hook program is executable by the `root` user: From 13ed5d710674a9467c8959fed12641b40e38c791 Mon Sep 17 00:00:00 2001 From: Ilan Joselevich Date: Tue, 26 Sep 2023 00:31:00 +0200 Subject: [PATCH 139/139] flakes: adopt repl-flake behavior as default --- doc/manual/src/release-notes/rl-next.md | 2 ++ src/libutil/experimental-features.cc | 2 ++ src/nix/repl.cc | 2 +- src/nix/repl.md | 2 +- tests/repl.sh | 11 +++-------- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 2467e7ccc..a69cad1db 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -3,3 +3,5 @@ - [URL flake references](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) now support [percent-encoded](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1) characters. - [Path-like flake references](@docroot@/command-ref/new-cli/nix3-flake.md#path-like-syntax) now accept arbitrary unicode characters (except `#` and `?`). + +- The experimental feature `repl-flake` is no longer needed, as its functionality is now part of the `flakes` experimental feature. To get the previous behavior, use the `--file/--expr` flags accordingly. diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 6d92222b9..203455b63 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -163,6 +163,8 @@ constexpr std::array xpFeatureDetails = {{ .tag = Xp::ReplFlake, .name = "repl-flake", .description = R"( + *Enabled with [`flakes`](#xp-feature-flakes) since 2.19* + Allow passing [installables](@docroot@/command-ref/new-cli/nix.md#installables) to `nix repl`, making its interface consistent with the other experimental commands. )", }, diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 9677c1b48..63fe3044b 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -47,7 +47,7 @@ struct CmdRepl : RawInstallablesCommand void applyDefaultInstallables(std::vector & rawInstallables) override { - if (!experimentalFeatureSettings.isEnabled(Xp::ReplFlake) && !(file) && rawInstallables.size() >= 1) { + if (!experimentalFeatureSettings.isEnabled(Xp::Flakes) && !(file) && rawInstallables.size() >= 1) { warn("future versions of Nix will require using `--file` to load a file"); if (rawInstallables.size() > 1) warn("more than one input file is not currently supported"); diff --git a/src/nix/repl.md b/src/nix/repl.md index 9dc61fd35..32c08e24b 100644 --- a/src/nix/repl.md +++ b/src/nix/repl.md @@ -36,7 +36,7 @@ R""( Loading Installable ''... Added 1 variables. - # nix repl --extra-experimental-features 'flakes repl-flake' nixpkgs + # nix repl --extra-experimental-features 'flakes' nixpkgs Loading Installable 'flake:nixpkgs#'... Added 5 variables. diff --git a/tests/repl.sh b/tests/repl.sh index bb8b60e50..1b779c1f5 100644 --- a/tests/repl.sh +++ b/tests/repl.sh @@ -105,18 +105,13 @@ testReplResponseNoRegex ' testReplResponse ' drvPath ' '".*-simple.drv"' \ -$testDir/simple.nix +--file $testDir/simple.nix testReplResponse ' drvPath ' '".*-simple.drv"' \ --file $testDir/simple.nix --experimental-features 'ca-derivations' -testReplResponse ' -drvPath -' '".*-simple.drv"' \ ---file $testDir/simple.nix --extra-experimental-features 'repl-flake ca-derivations' - mkdir -p flake && cat < flake/flake.nix { outputs = { self }: { @@ -130,7 +125,7 @@ EOF testReplResponse ' foo + baz ' "3" \ - ./flake ./flake\#bar --experimental-features 'flakes repl-flake' + ./flake ./flake\#bar --experimental-features 'flakes' # Test the `:reload` mechansim with flakes: # - Eval `./flake#changingThing` @@ -143,7 +138,7 @@ sleep 1 # Leave the repl the time to eval 'foo' sed -i 's/beforeChange/afterChange/' flake/flake.nix echo ":reload" echo "changingThing" -) | nix repl ./flake --experimental-features 'flakes repl-flake') +) | nix repl ./flake --experimental-features 'flakes') echo "$replResult" | grepQuiet -s beforeChange echo "$replResult" | grepQuiet -s afterChange