From 783a8341ee2dedb8fc0790e803a5f3d0362d67d9 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 16 Jul 2024 01:28:28 +0200 Subject: [PATCH 001/142] tests/functional: Support negative codes in expect, expectStderr --- tests/functional/common/vars-and-functions.sh | 6 ++++-- tests/functional/test-infra.sh | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/tests/functional/common/vars-and-functions.sh b/tests/functional/common/vars-and-functions.sh index 4316a30d5..e5cc04bb3 100644 --- a/tests/functional/common/vars-and-functions.sh +++ b/tests/functional/common/vars-and-functions.sh @@ -236,7 +236,8 @@ expect() { expected="$1" shift "$@" && res=0 || res="$?" - if [[ $res -ne $expected ]]; then + # also match "negative" codes, which wrap around to >127 + if [[ $res -ne $expected && $res -ne $[256 + expected] ]]; then echo "Expected exit code '$expected' but got '$res' from command ${*@Q}" >&2 return 1 fi @@ -250,7 +251,8 @@ expectStderr() { expected="$1" shift "$@" 2>&1 && res=0 || res="$?" - if [[ $res -ne $expected ]]; then + # also match "negative" codes, which wrap around to >127 + if [[ $res -ne $expected && $res -ne $[256 + expected] ]]; then echo "Expected exit code '$expected' but got '$res' from command ${*@Q}" >&2 return 1 fi diff --git a/tests/functional/test-infra.sh b/tests/functional/test-infra.sh index 37322b356..f6f84eae9 100755 --- a/tests/functional/test-infra.sh +++ b/tests/functional/test-infra.sh @@ -13,6 +13,25 @@ expect 1 false # `expect` will fail when we get it wrong expect 1 expect 0 false +function ret() { + return $1 +} + +# `expect` can call functions, not just executables +expect 0 ret 0 +expect 1 ret 1 + +# `expect` supports negative exit codes +expect -1 ret -1 + +# or high positive ones, equivalent to negative ones +expect 255 ret 255 +expect 255 ret -1 +expect -1 ret 255 + +# but it doesn't confuse negative exit codes with positive ones +expect 1 expect -10 ret 10 + noisyTrue () { echo YAY! >&2 true From f2df3f0c6c78cb742a87dbe2d2f9bcf5d5395795 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 16 Jul 2024 01:40:14 +0200 Subject: [PATCH 002/142] tests/vars-and-functions: Add callerPrefix helper --- tests/functional/common/vars-and-functions.sh | 39 +++++++++++++++++++ tests/functional/test-infra.sh | 4 ++ 2 files changed, 43 insertions(+) diff --git a/tests/functional/common/vars-and-functions.sh b/tests/functional/common/vars-and-functions.sh index e5cc04bb3..062f3d9f9 100644 --- a/tests/functional/common/vars-and-functions.sh +++ b/tests/functional/common/vars-and-functions.sh @@ -297,6 +297,45 @@ onError() { done } +# Prints an error message prefix referring to the last call into this file. +# Ignores `expect` and `expectStderr` calls. +# Set a special exit code when test suite functions are misused, so that +# functions like expectStderr won't mistake them for expected Nix CLI errors. +# Suggestion: -101 (negative to indicate very abnormal, and beyond the normal +# range of signals) +# Example (showns as string): 'repl.sh:123: in call to grepQuiet: ' +# This function is inefficient, so it should only be used in error messages. +callerPrefix() { + # Find the closes caller that's not from this file + local i file line fn savedFn + # Use `caller` + for i in $(seq 0 100); do + caller $i > /dev/null || { + if [[ -n "${file:-}" ]]; then + echo "$file:$line: ${savedFn+in call to $savedFn: }" + fi + break + } + line="$(caller $i | cut -d' ' -f1)" + fn="$(caller $i | cut -d' ' -f2)" + file="$(caller $i | cut -d' ' -f3)" + if [[ $file != "${BASH_SOURCE[0]}" ]]; then + echo "$file:$line: ${savedFn+in call to $savedFn: }" + return + fi + case "$fn" in + # Ignore higher order functions that don't report any misuse of themselves + # This way a misuse of a foo in `expectStderr 1 foo` will be reported as + # calling foo, not expectStderr. + expect|expectStderr|callerPrefix) + ;; + *) + savedFn="$fn" + ;; + esac + done +} + # `grep -v` doesn't work well for exit codes. We want `!(exist line l. l # matches)`. It gives us `exist line l. !(l matches)`. # diff --git a/tests/functional/test-infra.sh b/tests/functional/test-infra.sh index f6f84eae9..93e0bd64b 100755 --- a/tests/functional/test-infra.sh +++ b/tests/functional/test-infra.sh @@ -88,6 +88,10 @@ funBang () { expect 1 funBang unset funBang +# callerPrefix can be used by the test framework to improve error messages +# it reports about our call site here +echo "<[$(callerPrefix)]>" | grepQuiet -F "<[test-infra.sh:$LINENO: ]>" + # `grep -v -q` is not what we want for exit codes, but `grepInverse` is # Avoid `grep -v -q`. The following line proves the point, and if it fails, # we'll know that `grep` had a breaking change or `-v -q` may not be portable. From 644b97ce2574fe22a3fe14daeb6a3d0711d75731 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 16 Jul 2024 01:41:22 +0200 Subject: [PATCH 003/142] tests/functional: Make our grep* helpers reject newlines in the query Newlines behave like *OR*; not "and then". --- tests/functional/common/vars-and-functions.sh | 19 ++++++++++++++++--- tests/functional/test-infra.sh | 5 +++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/tests/functional/common/vars-and-functions.sh b/tests/functional/common/vars-and-functions.sh index 062f3d9f9..6cce08fbc 100644 --- a/tests/functional/common/vars-and-functions.sh +++ b/tests/functional/common/vars-and-functions.sh @@ -336,13 +336,24 @@ callerPrefix() { done } +checkGrepArgs() { + local arg + for arg in "$@"; do + if [[ "$arg" != "${arg//$'\n'/_}" ]]; then + echo "$(callerPrefix)newline not allowed in arguments; grep would try each line individually as if connected by an OR operator" >&2 + return -101 + fi + done +} + # `grep -v` doesn't work well for exit codes. We want `!(exist line l. l # matches)`. It gives us `exist line l. !(l matches)`. # # `!` normally doesn't work well with `set -e`, but when we wrap in a # function it *does*. grepInverse() { - ! grep "$@" + checkGrepArgs "$@" && \ + ! grep "$@" } # A shorthand, `> /dev/null` is a bit noisy. @@ -357,12 +368,14 @@ grepInverse() { # the producer into the pipe. But rest assured we've seen it happen in # CI reliably. grepQuiet() { - grep "$@" > /dev/null + checkGrepArgs "$@" && \ + grep "$@" > /dev/null } # The previous two, combined grepQuietInverse() { - ! grep "$@" > /dev/null + checkGrepArgs "$@" && \ + ! grep "$@" > /dev/null } # Return the number of arguments diff --git a/tests/functional/test-infra.sh b/tests/functional/test-infra.sh index 93e0bd64b..983b4e860 100755 --- a/tests/functional/test-infra.sh +++ b/tests/functional/test-infra.sh @@ -108,3 +108,8 @@ unset res res=$(set -eu -o pipefail; echo foo | expect 1 grepQuietInverse foo | wc -c) (( res == 0 )) unset res + +# `grepQuiet` does not allow newlines in its arguments, because grep quietly +# treats them as multiple queries. +( echo foo; echo bar; ) | expectStderr -101 grepQuiet $'foo\nbar' \ + | grepQuiet -E 'test-infra\.sh:[0-9]+: in call to grepQuiet: newline not allowed in arguments; grep would try each line individually as if connected by an OR operator' From 41a03738d63b94366d96dfc3e5cfb052c0ad5e2a Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 16 Jul 2024 01:54:12 +0200 Subject: [PATCH 004/142] tests/functional: Also keep plain grep calls safe from newlines --- tests/functional/common/vars-and-functions.sh | 20 ++++++++++++++++--- tests/functional/test-infra.sh | 4 ++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/tests/functional/common/vars-and-functions.sh b/tests/functional/common/vars-and-functions.sh index 6cce08fbc..7a399f6d4 100644 --- a/tests/functional/common/vars-and-functions.sh +++ b/tests/functional/common/vars-and-functions.sh @@ -351,9 +351,12 @@ checkGrepArgs() { # # `!` normally doesn't work well with `set -e`, but when we wrap in a # function it *does*. +# +# `command grep` lets us avoid re-checking the args by going directly to the +# executable. grepInverse() { checkGrepArgs "$@" && \ - ! grep "$@" + ! command grep "$@" } # A shorthand, `> /dev/null` is a bit noisy. @@ -367,15 +370,26 @@ grepInverse() { # the closing of the pipe, the buffering of the pipe, and the speed of # the producer into the pipe. But rest assured we've seen it happen in # CI reliably. +# +# `command grep` lets us avoid re-checking the args by going directly to the +# executable. grepQuiet() { checkGrepArgs "$@" && \ - grep "$@" > /dev/null + command grep "$@" > /dev/null } # The previous two, combined grepQuietInverse() { checkGrepArgs "$@" && \ - ! grep "$@" > /dev/null + ! command grep "$@" > /dev/null +} + +# Wrap grep to remove its newline footgun; see checkGrepArgs. +# Note that we keep the checkGrepArgs calls in the other helpers, because some +# of them are negated and that would defeat this check. +grep() { + checkGrepArgs "$@" && \ + command grep "$@" } # Return the number of arguments diff --git a/tests/functional/test-infra.sh b/tests/functional/test-infra.sh index 983b4e860..1dab069fb 100755 --- a/tests/functional/test-infra.sh +++ b/tests/functional/test-infra.sh @@ -113,3 +113,7 @@ unset res # treats them as multiple queries. ( echo foo; echo bar; ) | expectStderr -101 grepQuiet $'foo\nbar' \ | grepQuiet -E 'test-infra\.sh:[0-9]+: in call to grepQuiet: newline not allowed in arguments; grep would try each line individually as if connected by an OR operator' + +# We took the blue pill and woke up in a world where `grep` is moderately safe. +expectStderr -101 grep $'foo\nbar' \ + | grepQuiet -E 'test-infra\.sh:[0-9]+: in call to grep: newline not allowed in arguments; grep would try each line individually as if connected by an OR operator' From 74698d54c82fa302dc0c82f68c796eda673090d9 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 10 Jul 2024 13:34:42 +0200 Subject: [PATCH 005/142] Document builtins.derivation --- src/libexpr/primops/derivation.nix | 29 +++++++++++++++++-- .../lang/eval-fail-derivation-name.err.exp | 24 +++++++-------- 2 files changed, 39 insertions(+), 14 deletions(-) diff --git a/src/libexpr/primops/derivation.nix b/src/libexpr/primops/derivation.nix index c0fbe8082..f329ff71e 100644 --- a/src/libexpr/primops/derivation.nix +++ b/src/libexpr/primops/derivation.nix @@ -1,6 +1,31 @@ -/* This is the implementation of the ‘derivation’ builtin function. - It's actually a wrapper around the ‘derivationStrict’ primop. */ +# This is the implementation of the ‘derivation’ builtin function. +# It's actually a wrapper around the ‘derivationStrict’ primop. +# Note that the following comment will be shown in :doc in the repl, but not in the manual. +/** + Create a derivation. + + # Inputs + + The single argument is an attribute set that describes what to build and how to build it. + See https://nix.dev/manual/nix/2.23/language/derivations + + # Output + + The result is an attribute set that describes the derivation. + Notably it contains the outputs, which in the context of the Nix language are special strings that refer to the output paths, which may not yet exist. + The realisation of these outputs only occurs when needed; for example + + * When `nix-build` or a similar command is run, it realises the outputs that were requested on its command line. + See https://nix.dev/manual/nix/2.23/command-ref/nix-build + + * When `import`, `readFile`, `readDir` or some other functions are called, they have to realise the outputs they depend on. + This is referred to as "import from derivation". + See https://nix.dev/manual/nix/2.23/language/import-from-derivation + + Note that `derivation` is very bare-bones, and provides almost no commands during the build. + Most likely, you'll want to use functions like `stdenv.mkDerivation` in Nixpkgs to set up a basic environment. +*/ drvAttrs @ { outputs ? [ "out" ], ... }: let diff --git a/tests/functional/lang/eval-fail-derivation-name.err.exp b/tests/functional/lang/eval-fail-derivation-name.err.exp index eb2206df1..ae7b47712 100644 --- a/tests/functional/lang/eval-fail-derivation-name.err.exp +++ b/tests/functional/lang/eval-fail-derivation-name.err.exp @@ -1,24 +1,24 @@ error: … while evaluating the attribute 'outPath' - at :19:9: - 18| value = commonAttrs // { - 19| outPath = builtins.getAttr outputName strict; + at :44:9: + 43| value = commonAttrs // { + 44| outPath = builtins.getAttr outputName strict; | ^ - 20| drvPath = strict.drvPath; + 45| drvPath = strict.drvPath; … while calling the 'getAttr' builtin - at :19:19: - 18| value = commonAttrs // { - 19| outPath = builtins.getAttr outputName strict; + at :44:19: + 43| value = commonAttrs // { + 44| outPath = builtins.getAttr outputName strict; | ^ - 20| drvPath = strict.drvPath; + 45| drvPath = strict.drvPath; … while calling the 'derivationStrict' builtin - at :9:12: - 8| - 9| strict = derivationStrict drvAttrs; + at :34:12: + 33| + 34| strict = derivationStrict drvAttrs; | ^ - 10| + 35| … while evaluating derivation '~jiggle~' whose name attribute is located at /pwd/lang/eval-fail-derivation-name.nix:2:3 From f9a1d6b0188a2d0e9ad84f1e735e560320380501 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 16 Jul 2024 17:36:30 +0200 Subject: [PATCH 006/142] tests/functional/lang: Add post processing and remove certain line numbers --- tests/functional/lang.sh | 15 +++++++++++ .../lang/eval-fail-derivation-name.err.exp | 26 +++++++++---------- .../eval-fail-derivation-name.postprocess | 9 +++++++ 3 files changed, 37 insertions(+), 13 deletions(-) create mode 100755 tests/functional/lang/eval-fail-derivation-name.postprocess diff --git a/tests/functional/lang.sh b/tests/functional/lang.sh index 8cb8e98fb..5a8cefd13 100755 --- a/tests/functional/lang.sh +++ b/tests/functional/lang.sh @@ -50,11 +50,22 @@ set +x badDiff=0 badExitCode=0 +# Extra post-processing that's specific to each test case +postprocess() { + if [[ -e "lang/$1.postprocess" ]]; then + ( + set -x; + "lang/$1.postprocess" "lang/$1" + ) + fi +} + for i in lang/parse-fail-*.nix; do echo "parsing $i (should fail)"; i=$(basename "$i" .nix) if expectStderr 1 nix-instantiate --parse - < "lang/$i.nix" > "lang/$i.err" then + postprocess "$i" diffAndAccept "$i" err err.exp else echo "FAIL: $i shouldn't parse" @@ -71,6 +82,7 @@ for i in lang/parse-okay-*.nix; do 2> "lang/$i.err" then sed "s!$(pwd)!/pwd!g" "lang/$i.out" "lang/$i.err" + postprocess "$i" diffAndAccept "$i" out exp diffAndAccept "$i" err err.exp else @@ -94,6 +106,7 @@ for i in lang/eval-fail-*.nix; do expectStderr 1 nix-instantiate $flags "lang/$i.nix" \ | sed "s!$(pwd)!/pwd!g" > "lang/$i.err" then + postprocess "$i" diffAndAccept "$i" err err.exp else echo "FAIL: $i shouldn't evaluate" @@ -109,6 +122,7 @@ for i in lang/eval-okay-*.nix; do if expect 0 nix-instantiate --eval --xml --no-location --strict \ "lang/$i.nix" > "lang/$i.out.xml" then + postprocess "$i" diffAndAccept "$i" out.xml exp.xml else echo "FAIL: $i should evaluate" @@ -129,6 +143,7 @@ for i in lang/eval-okay-*.nix; do 2> "lang/$i.err" then sed -i "s!$(pwd)!/pwd!g" "lang/$i.out" "lang/$i.err" + postprocess "$i" diffAndAccept "$i" out exp diffAndAccept "$i" err err.exp else diff --git a/tests/functional/lang/eval-fail-derivation-name.err.exp b/tests/functional/lang/eval-fail-derivation-name.err.exp index ae7b47712..0ef98674d 100644 --- a/tests/functional/lang/eval-fail-derivation-name.err.exp +++ b/tests/functional/lang/eval-fail-derivation-name.err.exp @@ -1,26 +1,26 @@ error: … while evaluating the attribute 'outPath' - at :44:9: - 43| value = commonAttrs // { - 44| outPath = builtins.getAttr outputName strict; + at ::: + | value = commonAttrs // { + | outPath = builtins.getAttr outputName strict; | ^ - 45| drvPath = strict.drvPath; + | drvPath = strict.drvPath; … while calling the 'getAttr' builtin - at :44:19: - 43| value = commonAttrs // { - 44| outPath = builtins.getAttr outputName strict; + at ::: + | value = commonAttrs // { + | outPath = builtins.getAttr outputName strict; | ^ - 45| drvPath = strict.drvPath; + | drvPath = strict.drvPath; … while calling the 'derivationStrict' builtin - at :34:12: - 33| - 34| strict = derivationStrict drvAttrs; + at ::: + | + | strict = derivationStrict drvAttrs; | ^ - 35| + | … while evaluating derivation '~jiggle~' - whose name attribute is located at /pwd/lang/eval-fail-derivation-name.nix:2:3 + whose name attribute is located at /pwd/lang/eval-fail-derivation-name.nix:: error: invalid derivation name: name '~jiggle~' contains illegal character '~'. Please pass a different 'name'. diff --git a/tests/functional/lang/eval-fail-derivation-name.postprocess b/tests/functional/lang/eval-fail-derivation-name.postprocess new file mode 100755 index 000000000..ab9fa5b5d --- /dev/null +++ b/tests/functional/lang/eval-fail-derivation-name.postprocess @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail +testcaseBasename=$1 + +# Line numbers change when derivation.nix docs are updated. +sed -i "$testcaseBasename.err" \ + -e 's/[0-9 ][0-9 ][0-9 ][0-9 ][0-9 ][0-9 ][0-9 ][0-9]\([^0-9]\)/\1/g' \ + -e 's/[0-9][0-9]*//g' \ + ; From 7dce07463429bb4a42a3b47a869f5eb488206c26 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 16 Jul 2024 22:43:04 +0200 Subject: [PATCH 007/142] tests/functional/lang: Avoid /usr/bin/env for sandbox --- tests/functional/lang.sh | 5 ++++- tests/functional/lang/eval-fail-derivation-name.postprocess | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) mode change 100755 => 100644 tests/functional/lang/eval-fail-derivation-name.postprocess diff --git a/tests/functional/lang.sh b/tests/functional/lang.sh index 5a8cefd13..46cf3f1fe 100755 --- a/tests/functional/lang.sh +++ b/tests/functional/lang.sh @@ -54,8 +54,11 @@ badExitCode=0 postprocess() { if [[ -e "lang/$1.postprocess" ]]; then ( + # We could allow arbitrary interpreters in .postprocess, but that + # just exposes us to the complexity of not having /usr/bin/env in + # the sandbox. So let's just hardcode bash for now. set -x; - "lang/$1.postprocess" "lang/$1" + bash "lang/$1.postprocess" "lang/$1" ) fi } diff --git a/tests/functional/lang/eval-fail-derivation-name.postprocess b/tests/functional/lang/eval-fail-derivation-name.postprocess old mode 100755 new mode 100644 index ab9fa5b5d..ffbc2b5d4 --- a/tests/functional/lang/eval-fail-derivation-name.postprocess +++ b/tests/functional/lang/eval-fail-derivation-name.postprocess @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +# shellcheck shell=bash set -euo pipefail testcaseBasename=$1 From 83d585b423737591f865d905882122211995c308 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 17 Jul 2024 15:19:17 +0200 Subject: [PATCH 008/142] C API: Make nix_err an enum This generally gives a better experience with bindings generators, possibly other tooling. A possible risk is that some generators may not represent unknown codes correctly. Rust bindgen by default generates suitable code: * a type alias nix_err = c_int * individual constants for the known enum values It does _not_ generate a closed type that can only hold the values that were known at code generation time. If this proves to be a problem, we could instead split the type: `typedef int nix_err;` for return values `enum nix_known_err` for code generation. This would complicate the interface, so let's not do it unless it is shown to be needed. --- src/libutil-c/nix_api_util.h | 76 +++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 36 deletions(-) diff --git a/src/libutil-c/nix_api_util.h b/src/libutil-c/nix_api_util.h index e0ca04e69..ad6f32859 100644 --- a/src/libutil-c/nix_api_util.h +++ b/src/libutil-c/nix_api_util.h @@ -56,47 +56,51 @@ extern "C" { * - NIX_ERR_KEY: A key error occurred (-3) * - NIX_ERR_NIX_ERROR: A generic Nix error occurred (-4) */ -typedef int nix_err; +enum nix_err { -/** - * @brief No error occurred. - * - * This error code is returned when no error has occurred during the function - * execution. - */ -#define NIX_OK 0 + /** + * @brief No error occurred. + * + * This error code is returned when no error has occurred during the function + * execution. + */ + NIX_OK = 0, -/** - * @brief An unknown error occurred. - * - * This error code is returned when an unknown error occurred during the - * function execution. - */ -#define NIX_ERR_UNKNOWN -1 + /** + * @brief An unknown error occurred. + * + * This error code is returned when an unknown error occurred during the + * function execution. + */ + NIX_ERR_UNKNOWN = -1, -/** - * @brief An overflow error occurred. - * - * This error code is returned when an overflow error occurred during the - * function execution. - */ -#define NIX_ERR_OVERFLOW -2 + /** + * @brief An overflow error occurred. + * + * This error code is returned when an overflow error occurred during the + * function execution. + */ + NIX_ERR_OVERFLOW = -2, -/** - * @brief A key error occurred. - * - * This error code is returned when a key error occurred during the function - * execution. - */ -#define NIX_ERR_KEY -3 + /** + * @brief A key error occurred. + * + * This error code is returned when a key error occurred during the function + * execution. + */ + NIX_ERR_KEY = -3, -/** - * @brief A generic Nix error occurred. - * - * This error code is returned when a generic Nix error occurred during the - * function execution. - */ -#define NIX_ERR_NIX_ERROR -4 + /** + * @brief A generic Nix error occurred. + * + * This error code is returned when a generic Nix error occurred during the + * function execution. + */ + NIX_ERR_NIX_ERROR = -4, + +}; + +typedef enum nix_err nix_err; /** * @brief This object stores error state. From 4457cebe05c444facc68db281c58f90a87f785ce Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 23 Jul 2024 10:24:18 +0200 Subject: [PATCH 009/142] Update comment in tests//vars-and-functions.sh Co-authored-by: tomberek --- tests/functional/common/vars-and-functions.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/functional/common/vars-and-functions.sh b/tests/functional/common/vars-and-functions.sh index 7a399f6d4..954435593 100644 --- a/tests/functional/common/vars-and-functions.sh +++ b/tests/functional/common/vars-and-functions.sh @@ -306,7 +306,8 @@ onError() { # Example (showns as string): 'repl.sh:123: in call to grepQuiet: ' # This function is inefficient, so it should only be used in error messages. callerPrefix() { - # Find the closes caller that's not from this file + # Find the closest caller that's not from this file + # using the bash `caller` builtin. local i file line fn savedFn # Use `caller` for i in $(seq 0 100); do From 498eed0a25ee1707677d56e0ea3628584991453b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 23 Jul 2024 10:56:29 +0200 Subject: [PATCH 010/142] ~FramedSource(): Don't throw an exception if the remote has disconnected This would cause the daemon to crash with a call to terminate(). --- src/libutil/serialise.hh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 8254a5f89..8137db5f4 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -483,13 +483,17 @@ struct FramedSource : Source ~FramedSource() { - if (!eof) { - while (true) { - auto n = readInt(from); - if (!n) break; - std::vector data(n); - from(data.data(), n); + try { + if (!eof) { + while (true) { + auto n = readInt(from); + if (!n) break; + std::vector data(n); + from(data.data(), n); + } } + } catch (...) { + ignoreException(); } } From eb89e50cbb4584bb42a425089640b2931b723cca Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 23 Jul 2024 16:21:43 +0200 Subject: [PATCH 011/142] Rejiggle getGCCycles() for buildNoGc --- src/libexpr/eval-gc.cc | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/libexpr/eval-gc.cc b/src/libexpr/eval-gc.cc index 97b24e3fb..73ab809e5 100644 --- a/src/libexpr/eval-gc.cc +++ b/src/libexpr/eval-gc.cc @@ -206,10 +206,17 @@ static inline void initGCReal() } } +static size_t gcCyclesAfterInit = 0; + +size_t getGCCycles() +{ + assertGCInitialized(); + return static_cast(GC_get_gc_no()) - gcCyclesAfterInit; +} + #endif static bool gcInitialised = false; -static GC_word gcCyclesAfterInit = 0; void initGC() { @@ -218,10 +225,11 @@ void initGC() #if HAVE_BOEHMGC initGCReal(); + + gcCyclesAfterInit = GC_get_gc_no(); #endif gcInitialised = true; - gcCyclesAfterInit = GC_get_gc_no(); } void assertGCInitialized() @@ -229,10 +237,4 @@ void assertGCInitialized() assert(gcInitialised); } -size_t getGCCycles() -{ - assertGCInitialized(); - return GC_get_gc_no() - gcCyclesAfterInit; -} - } // namespace nix From 5d6bc484beab6ff822815b5b26745710d8c86305 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 23 Jul 2024 16:24:43 +0200 Subject: [PATCH 012/142] Hide getGCCycles when we have no GC Alternatively, we could make it return 0, but we don't need it in the first place because the caller exists conditionally too. --- src/libexpr/eval-gc.hh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libexpr/eval-gc.hh b/src/libexpr/eval-gc.hh index 76f019729..005175eb7 100644 --- a/src/libexpr/eval-gc.hh +++ b/src/libexpr/eval-gc.hh @@ -15,9 +15,11 @@ void initGC(); */ void assertGCInitialized(); +#ifdef HAVE_BOEHMGC /** * The number of GC cycles since initGC(). */ size_t getGCCycles(); +#endif -} +} // namespace nix From 2b4e3f04a433d96e779fb91fcd9fa01329413d02 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 23 Jul 2024 16:57:45 +0200 Subject: [PATCH 013/142] markdown.cc: Format Slightly custom because the automated formatting messes up the braced initializer with named fields. --- src/libcmd/markdown.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index 88c3f640b..882f95faa 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -4,8 +4,8 @@ #include "terminal.hh" #if HAVE_LOWDOWN -# include -# include +# include +# include #endif namespace nix { @@ -15,7 +15,8 @@ std::string renderMarkdownToTerminal(std::string_view markdown) #if HAVE_LOWDOWN int windowWidth = getWindowSize().second; - struct lowdown_opts opts { + struct lowdown_opts opts + { .type = LOWDOWN_TERM, .maxdepth = 20, .cols = (size_t) std::max(windowWidth - 5, 60), From 97b0114ab84144e3574c74c7fb0e0e38681f3b6a Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 23 Jul 2024 16:54:04 +0200 Subject: [PATCH 014/142] renderMarkdownToTerminal: Add _NIX_TEST_RAW_MARKDOWN env var For testing only. --- src/libcmd/markdown.cc | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index 882f95faa..6a0d05d9f 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -1,5 +1,6 @@ #include "markdown.hh" -#include "util.hh" +#include "environment-variables.hh" +#include "error.hh" #include "finally.hh" #include "terminal.hh" @@ -10,9 +11,9 @@ namespace nix { -std::string renderMarkdownToTerminal(std::string_view markdown) -{ #if HAVE_LOWDOWN +static std::string doRenderMarkdownToTerminal(std::string_view markdown) +{ int windowWidth = getWindowSize().second; struct lowdown_opts opts @@ -52,9 +53,21 @@ std::string renderMarkdownToTerminal(std::string_view markdown) throw Error("allocation error while rendering Markdown"); return filterANSIEscapes(std::string(buf->data, buf->size), !isTTY()); -#else - return std::string(markdown); -#endif } +std::string renderMarkdownToTerminal(std::string_view markdown) +{ + if (auto e = getEnv("_NIX_TEST_RAW_MARKDOWN"); e && *e == "1") + return std::string(markdown); + else + return doRenderMarkdownToTerminal(markdown); } + +#else +std::string renderMarkdownToTerminal(std::string_view markdown) +{ + return std::string(markdown); +} +#endif + +} // namespace nix From 712ce2feacdc0cde7249ae6da5f8063858e80ae2 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 23 Jul 2024 17:50:51 +0200 Subject: [PATCH 015/142] ReadlineLikeInteracter::getLine: Add _NIX_TEST_REPL_ECHO env var ... for testing --- src/libcmd/repl-interacter.cc | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc index b285c8a9a..a12e7d8c7 100644 --- a/src/libcmd/repl-interacter.cc +++ b/src/libcmd/repl-interacter.cc @@ -19,6 +19,7 @@ extern "C" { #include "repl-interacter.hh" #include "file-system.hh" #include "repl.hh" +#include "environment-variables.hh" namespace nix { @@ -179,6 +180,19 @@ bool ReadlineLikeInteracter::getLine(std::string & input, ReplPromptType promptT return false; input += s; input += '\n'; + +#ifndef USE_READLINE + // editline doesn't echo the input to the output when non-interactive, unlike readline + // this results in a different behavior when running tests. The echoing is + // quite useful for reading the test output, so we add it here. + if (auto e = getEnv("_NIX_TEST_REPL_ECHO"); s && e && *e == "1") + { + // This is probably not right for multi-line input, but we don't use that + // in the characterisation tests, so it's fine. + std::cout << "nix-repl> " << s << std::endl; + } +#endif + return true; } From ca2cc26e12d7264fb141206e0bc35ade2378b8f8 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 23 Jul 2024 17:52:28 +0200 Subject: [PATCH 016/142] tests/functional/repl: Improve precision and readability ... as well as match buildReadlineNoMarkdown. Unfortunately it doesn't support long inputs or multiline inputs for now. This needs to make better use of the interacter interface. --- src/libcmd/repl-interacter.cc | 16 ++--- tests/functional/repl.sh | 11 ++++ .../repl/doc-comment-curried-args.expected | 26 ++++---- .../repl/doc-comment-curried-args.in | 2 +- .../repl/doc-comment-formals.expected | 13 ++-- .../repl/doc-comment-function.expected | 7 ++- tests/functional/repl/doc-compact.expected | 12 ++-- tests/functional/repl/doc-constant.expected | 60 +++++++++++-------- tests/functional/repl/doc-floatedIn.expected | 12 ++-- .../repl/doc-lambda-flavors.expected | 36 +++++------ .../functional/repl/doc-measurement.expected | 12 ++-- tests/functional/repl/doc-multiply.expected | 22 ++++--- .../functional/repl/doc-unambiguous.expected | 12 ++-- .../repl/pretty-print-idempotent.expected | 12 +++- 14 files changed, 154 insertions(+), 99 deletions(-) diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc index a12e7d8c7..76fe38780 100644 --- a/src/libcmd/repl-interacter.cc +++ b/src/libcmd/repl-interacter.cc @@ -176,22 +176,22 @@ bool ReadlineLikeInteracter::getLine(std::string & input, ReplPromptType promptT return true; } - if (!s) - return false; - input += s; - input += '\n'; - -#ifndef USE_READLINE // editline doesn't echo the input to the output when non-interactive, unlike readline // this results in a different behavior when running tests. The echoing is // quite useful for reading the test output, so we add it here. if (auto e = getEnv("_NIX_TEST_REPL_ECHO"); s && e && *e == "1") { +#ifndef USE_READLINE // This is probably not right for multi-line input, but we don't use that // in the characterisation tests, so it's fine. - std::cout << "nix-repl> " << s << std::endl; - } + std::cout << promptForType(promptType) << s << std::endl; #endif + } + + if (!s) + return false; + input += s; + input += '\n'; return true; } diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index 4f5bb36aa..a6c075abd 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -262,6 +262,14 @@ badExitCode=0 nixVersion="$(nix eval --impure --raw --expr 'builtins.nixVersion' --extra-experimental-features nix-command)" +# I couldn't get readline and editline to agree on the newline before the prompt, +# so let's just force it to be one empty line. Ideally we get the two to agree +# or use a simpler interacter for testing. +stripEmptyLinesBeforePrompt() { + # --null-data: treat input as NUL-terminated instead of newline-terminated + sed --null-data 's/\n\n*nix-repl>/\n\nnix-repl>/g' +} + runRepl () { # That is right, we are also filtering out the testdir _without underscores_. @@ -273,8 +281,11 @@ runRepl () { testDirNoUnderscores="${testDir//_/}" # TODO: pass arguments to nix repl; see lang.sh + _NIX_TEST_RAW_MARKDOWN=1 \ + _NIX_TEST_REPL_ECHO=1 \ nix repl 2>&1 \ | stripColors \ + | stripEmptyLinesBeforePrompt \ | sed \ -e "s@$testDir@/path/to/tests/functional@g" \ -e "s@$testDirNoUnderscores@/path/to/tests/functional@g" \ diff --git a/tests/functional/repl/doc-comment-curried-args.expected b/tests/functional/repl/doc-comment-curried-args.expected index c10c171e1..f97f593c6 100644 --- a/tests/functional/repl/doc-comment-curried-args.expected +++ b/tests/functional/repl/doc-comment-curried-args.expected @@ -1,24 +1,30 @@ Nix Type :? for help. + +nix-repl> :l doc-comments.nix Added variables. -Function curriedArgs - … defined at - /path/to/tests/functional/repl/doc-comments.nix:48:5 +nix-repl> :doc curriedArgs +Function `curriedArgs`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:48:5 - A documented function. +A documented function. +nix-repl> x = curriedArgs 1 +nix-repl> "Note that users may not expect this to behave as it currently does" "Note that users may not expect this to behave as it currently does" -Function curriedArgs - … defined at - /path/to/tests/functional/repl/doc-comments.nix:50:5 +nix-repl> :doc x +Function `curriedArgs`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:50:5 - The function returned by applying once +The function returned by applying once -"This won't produce documentation, because we can't actually document arbitrary values" +nix-repl> "This won't produce docs; no support for arbitrary values" +"This won't produce docs; no support for arbitrary values" +nix-repl> :doc x 2 error: value does not have documentation - +nix-repl> diff --git a/tests/functional/repl/doc-comment-curried-args.in b/tests/functional/repl/doc-comment-curried-args.in index 8dbbfc370..06ba21dcc 100644 --- a/tests/functional/repl/doc-comment-curried-args.in +++ b/tests/functional/repl/doc-comment-curried-args.in @@ -3,5 +3,5 @@ x = curriedArgs 1 "Note that users may not expect this to behave as it currently does" :doc x -"This won't produce documentation, because we can't actually document arbitrary values" +"This won't produce docs; no support for arbitrary values" :doc x 2 diff --git a/tests/functional/repl/doc-comment-formals.expected b/tests/functional/repl/doc-comment-formals.expected index 704c0050b..4c9757f89 100644 --- a/tests/functional/repl/doc-comment-formals.expected +++ b/tests/functional/repl/doc-comment-formals.expected @@ -1,13 +1,16 @@ Nix Type :? for help. + +nix-repl> :l doc-comments.nix Added variables. +nix-repl> "Note that this is not yet complete" "Note that this is not yet complete" -Function documentedFormals - … defined at - /path/to/tests/functional/repl/doc-comments.nix:57:5 - - Finds x +nix-repl> :doc documentedFormals +Function `documentedFormals`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:57:5 +Finds x +nix-repl> diff --git a/tests/functional/repl/doc-comment-function.expected b/tests/functional/repl/doc-comment-function.expected index 5ec465a96..9718ccc66 100644 --- a/tests/functional/repl/doc-comment-function.expected +++ b/tests/functional/repl/doc-comment-function.expected @@ -1,8 +1,9 @@ Nix Type :? for help. -Function defined at - /path/to/tests/functional/repl/doc-comment-function.nix:2:1 - A doc comment for a file that only contains a function +nix-repl> :doc import ./doc-comment-function.nix +Function defined at /path/to/tests/functional/repl/doc-comment-function.nix:2:1 +A doc comment for a file that only contains a function +nix-repl> diff --git a/tests/functional/repl/doc-compact.expected b/tests/functional/repl/doc-compact.expected index 4b05b653c..17603d813 100644 --- a/tests/functional/repl/doc-compact.expected +++ b/tests/functional/repl/doc-compact.expected @@ -1,11 +1,13 @@ Nix Type :? for help. + +nix-repl> :l doc-comments.nix Added variables. -Function compact - … defined at - /path/to/tests/functional/repl/doc-comments.nix:18:20 - - boom +nix-repl> :doc compact +Function `compact`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:18:20 +boom +nix-repl> diff --git a/tests/functional/repl/doc-constant.expected b/tests/functional/repl/doc-constant.expected index c66558333..e0092453e 100644 --- a/tests/functional/repl/doc-constant.expected +++ b/tests/functional/repl/doc-constant.expected @@ -1,23 +1,27 @@ Nix Type :? for help. + +nix-repl> :l doc-comments.nix Added variables. +nix-repl> :doc constant error: value does not have documentation -Attribute version +nix-repl> :doc lib.version +Attribute `version` - … defined at - /path/to/tests/functional/repl/doc-comments.nix:30:3 + … defined at /path/to/tests/functional/repl/doc-comments.nix:30:3 - Immovably fixed. +Immovably fixed. -Attribute empty +nix-repl> :doc lib.attr.empty +Attribute `empty` - … defined at - /path/to/tests/functional/repl/doc-comments.nix:33:3 + … defined at /path/to/tests/functional/repl/doc-comments.nix:33:3 - Unchangeably constant. +Unchangeably constant. +nix-repl> :doc lib.attr.undocument error: … while evaluating the attribute 'attr.undocument' at /path/to/tests/functional/repl/doc-comments.nix:33:3: @@ -32,59 +36,65 @@ error: | ^ Did you mean undocumented? -Attribute constant +nix-repl> :doc (import ./doc-comments.nix).constant +Attribute `constant` - … defined at - /path/to/tests/functional/repl/doc-comments.nix:27:3 + … defined at /path/to/tests/functional/repl/doc-comments.nix:27:3 - Firmly rigid. +Firmly rigid. -Attribute version +nix-repl> :doc (import ./doc-comments.nix).lib.version +Attribute `version` - … defined at - /path/to/tests/functional/repl/doc-comments.nix:30:3 + … defined at /path/to/tests/functional/repl/doc-comments.nix:30:3 - Immovably fixed. +Immovably fixed. -Attribute empty +nix-repl> :doc (import ./doc-comments.nix).lib.attr.empty +Attribute `empty` - … defined at - /path/to/tests/functional/repl/doc-comments.nix:33:3 + … defined at /path/to/tests/functional/repl/doc-comments.nix:33:3 - Unchangeably constant. +Unchangeably constant. -Attribute undocumented +nix-repl> :doc (import ./doc-comments.nix).lib.attr.undocumented +Attribute `undocumented` - … defined at - /path/to/tests/functional/repl/doc-comments.nix:35:3 + … defined at /path/to/tests/functional/repl/doc-comments.nix:35:3 - No documentation found. +No documentation found. +nix-repl> :doc missing error: undefined variable 'missing' at «string»:1:1: 1| missing | ^ +nix-repl> :doc constanz error: undefined variable 'constanz' at «string»:1:1: 1| constanz | ^ +nix-repl> :doc missing.attr error: undefined variable 'missing' at «string»:1:1: 1| missing.attr | ^ +nix-repl> :doc lib.missing error: attribute 'missing' missing at «string»:1:1: 1| lib.missing | ^ +nix-repl> :doc lib.missing.attr error: attribute 'missing' missing at «string»:1:1: 1| lib.missing.attr | ^ +nix-repl> :doc lib.attr.undocumental error: … while evaluating the attribute 'attr.undocumental' at /path/to/tests/functional/repl/doc-comments.nix:33:3: @@ -99,4 +109,4 @@ error: | ^ Did you mean undocumented? - +nix-repl> diff --git a/tests/functional/repl/doc-floatedIn.expected b/tests/functional/repl/doc-floatedIn.expected index 30f135725..d3f1c3f65 100644 --- a/tests/functional/repl/doc-floatedIn.expected +++ b/tests/functional/repl/doc-floatedIn.expected @@ -1,11 +1,13 @@ Nix Type :? for help. + +nix-repl> :l doc-comments.nix Added variables. -Function floatedIn - … defined at - /path/to/tests/functional/repl/doc-comments.nix:16:5 - - This also works. +nix-repl> :doc floatedIn +Function `floatedIn`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:16:5 +This also works. +nix-repl> diff --git a/tests/functional/repl/doc-lambda-flavors.expected b/tests/functional/repl/doc-lambda-flavors.expected index 43f483ce9..268cac05e 100644 --- a/tests/functional/repl/doc-lambda-flavors.expected +++ b/tests/functional/repl/doc-lambda-flavors.expected @@ -1,29 +1,31 @@ Nix Type :? for help. + +nix-repl> :l doc-comments.nix Added variables. -Function nonStrict - … defined at - /path/to/tests/functional/repl/doc-comments.nix:37:70 +nix-repl> :doc nonStrict +Function `nonStrict`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:37:70 - My syntax is not strict, but I'm strict anyway. +My syntax is not strict, but I'm strict anyway. -Function strict - … defined at - /path/to/tests/functional/repl/doc-comments.nix:38:63 +nix-repl> :doc strict +Function `strict`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:38:63 - I don't have to be strict, but I am anyway. +I don't have to be strict, but I am anyway. -Function strictPre - … defined at - /path/to/tests/functional/repl/doc-comments.nix:40:48 +nix-repl> :doc strictPre +Function `strictPre`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:40:48 - Here's one way to do this +Here's one way to do this -Function strictPost - … defined at - /path/to/tests/functional/repl/doc-comments.nix:41:53 - - Here's another way to do this +nix-repl> :doc strictPost +Function `strictPost`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:41:53 +Here's another way to do this +nix-repl> diff --git a/tests/functional/repl/doc-measurement.expected b/tests/functional/repl/doc-measurement.expected index 8598aaedb..14ae11e80 100644 --- a/tests/functional/repl/doc-measurement.expected +++ b/tests/functional/repl/doc-measurement.expected @@ -1,11 +1,13 @@ Nix Type :? for help. + +nix-repl> :l doc-comments.nix Added variables. -Function measurement - … defined at - /path/to/tests/functional/repl/doc-comments.nix:13:17 - - 👈 precisely this wide 👉 +nix-repl> :doc measurement +Function `measurement`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:13:17 +👈 precisely this wide 👉 +nix-repl> diff --git a/tests/functional/repl/doc-multiply.expected b/tests/functional/repl/doc-multiply.expected index db82af91f..5cefda516 100644 --- a/tests/functional/repl/doc-multiply.expected +++ b/tests/functional/repl/doc-multiply.expected @@ -1,15 +1,19 @@ Nix Type :? for help. + +nix-repl> :l doc-comments.nix Added variables. -Function multiply - … defined at - /path/to/tests/functional/repl/doc-comments.nix:10:14 - - Perform arithmetic multiplication. It's kind of like - repeated addition, very neat. - - | multiply 2 3 - | => 6 +nix-repl> :doc multiply +Function `multiply`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:10:14 +Perform *arithmetic* multiplication. It's kind of like repeated **addition**, very neat. + +```nix +multiply 2 3 +=> 6 +``` + +nix-repl> diff --git a/tests/functional/repl/doc-unambiguous.expected b/tests/functional/repl/doc-unambiguous.expected index 825aa1ee1..0872750a1 100644 --- a/tests/functional/repl/doc-unambiguous.expected +++ b/tests/functional/repl/doc-unambiguous.expected @@ -1,11 +1,13 @@ Nix Type :? for help. + +nix-repl> :l doc-comments.nix Added variables. -Function unambiguous - … defined at - /path/to/tests/functional/repl/doc-comments.nix:24:5 - - Very close +nix-repl> :doc unambiguous +Function `unambiguous`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:24:5 +Very close +nix-repl> diff --git a/tests/functional/repl/pretty-print-idempotent.expected b/tests/functional/repl/pretty-print-idempotent.expected index f38b9b569..f09aec88d 100644 --- a/tests/functional/repl/pretty-print-idempotent.expected +++ b/tests/functional/repl/pretty-print-idempotent.expected @@ -1,29 +1,39 @@ Nix Type :? for help. + +nix-repl> :l pretty-print-idempotent.nix Added variables. +nix-repl> oneDeep { homepage = "https://example.com"; } +nix-repl> oneDeep { homepage = "https://example.com"; } +nix-repl> twoDeep { layerOne = { ... }; } +nix-repl> twoDeep { layerOne = { ... }; } +nix-repl> oneDeepList [ "https://example.com" ] +nix-repl> oneDeepList [ "https://example.com" ] +nix-repl> twoDeepList [ [ ... ] ] +nix-repl> twoDeepList [ [ ... ] ] - +nix-repl> From c4ae9bb45b72c852ecff8bf5d5153951116e2039 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 24 Jul 2024 01:01:00 +0200 Subject: [PATCH 017/142] tests/functional/repl: Normalize final prompt --- tests/functional/repl.sh | 15 +++++++++++++-- .../repl/doc-comment-curried-args.expected | 2 -- .../functional/repl/doc-comment-formals.expected | 2 -- .../functional/repl/doc-comment-function.expected | 2 -- tests/functional/repl/doc-compact.expected | 2 -- tests/functional/repl/doc-constant.expected | 2 -- tests/functional/repl/doc-floatedIn.expected | 2 -- tests/functional/repl/doc-lambda-flavors.expected | 2 -- tests/functional/repl/doc-measurement.expected | 2 -- tests/functional/repl/doc-multiply.expected | 2 -- tests/functional/repl/doc-unambiguous.expected | 2 -- .../repl/pretty-print-idempotent.expected | 2 -- 12 files changed, 13 insertions(+), 24 deletions(-) diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index a6c075abd..a149916ac 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -262,14 +262,23 @@ badExitCode=0 nixVersion="$(nix eval --impure --raw --expr 'builtins.nixVersion' --extra-experimental-features nix-command)" +# TODO: write a repl interacter for testing. Papering over the differences between readline / editline and between platforms is a pain. + # I couldn't get readline and editline to agree on the newline before the prompt, -# so let's just force it to be one empty line. Ideally we get the two to agree -# or use a simpler interacter for testing. +# so let's just force it to be one empty line. stripEmptyLinesBeforePrompt() { # --null-data: treat input as NUL-terminated instead of newline-terminated sed --null-data 's/\n\n*nix-repl>/\n\nnix-repl>/g' } +# We don't get a final prompt on darwin, so we strip this as well. +stripFinalPrompt() { + # Strip the final prompt and/or any trailing spaces + sed --null-data \ + -e 's/\(.*[^\n]\)\n\n*nix-repl>[ \n]*$/\1/' \ + -e 's/[ \n]*$/\n/' +} + runRepl () { # That is right, we are also filtering out the testdir _without underscores_. @@ -285,7 +294,9 @@ runRepl () { _NIX_TEST_REPL_ECHO=1 \ nix repl 2>&1 \ | stripColors \ + | tr -d '\0' \ | stripEmptyLinesBeforePrompt \ + | stripFinalPrompt \ | sed \ -e "s@$testDir@/path/to/tests/functional@g" \ -e "s@$testDirNoUnderscores@/path/to/tests/functional@g" \ diff --git a/tests/functional/repl/doc-comment-curried-args.expected b/tests/functional/repl/doc-comment-curried-args.expected index f97f593c6..56607e911 100644 --- a/tests/functional/repl/doc-comment-curried-args.expected +++ b/tests/functional/repl/doc-comment-curried-args.expected @@ -26,5 +26,3 @@ nix-repl> "This won't produce docs; no support for arbitrary values" nix-repl> :doc x 2 error: value does not have documentation - -nix-repl> diff --git a/tests/functional/repl/doc-comment-formals.expected b/tests/functional/repl/doc-comment-formals.expected index 4c9757f89..1024919f4 100644 --- a/tests/functional/repl/doc-comment-formals.expected +++ b/tests/functional/repl/doc-comment-formals.expected @@ -12,5 +12,3 @@ Function `documentedFormals`\ … defined at /path/to/tests/functional/repl/doc-comments.nix:57:5 Finds x - -nix-repl> diff --git a/tests/functional/repl/doc-comment-function.expected b/tests/functional/repl/doc-comment-function.expected index 9718ccc66..3889c4f78 100644 --- a/tests/functional/repl/doc-comment-function.expected +++ b/tests/functional/repl/doc-comment-function.expected @@ -5,5 +5,3 @@ nix-repl> :doc import ./doc-comment-function.nix Function defined at /path/to/tests/functional/repl/doc-comment-function.nix:2:1 A doc comment for a file that only contains a function - -nix-repl> diff --git a/tests/functional/repl/doc-compact.expected b/tests/functional/repl/doc-compact.expected index 17603d813..79f1fd44f 100644 --- a/tests/functional/repl/doc-compact.expected +++ b/tests/functional/repl/doc-compact.expected @@ -9,5 +9,3 @@ Function `compact`\ … defined at /path/to/tests/functional/repl/doc-comments.nix:18:20 boom - -nix-repl> diff --git a/tests/functional/repl/doc-constant.expected b/tests/functional/repl/doc-constant.expected index e0092453e..5787e04dc 100644 --- a/tests/functional/repl/doc-constant.expected +++ b/tests/functional/repl/doc-constant.expected @@ -108,5 +108,3 @@ error: 1| lib.attr.undocumental | ^ Did you mean undocumented? - -nix-repl> diff --git a/tests/functional/repl/doc-floatedIn.expected b/tests/functional/repl/doc-floatedIn.expected index d3f1c3f65..82bb80b95 100644 --- a/tests/functional/repl/doc-floatedIn.expected +++ b/tests/functional/repl/doc-floatedIn.expected @@ -9,5 +9,3 @@ Function `floatedIn`\ … defined at /path/to/tests/functional/repl/doc-comments.nix:16:5 This also works. - -nix-repl> diff --git a/tests/functional/repl/doc-lambda-flavors.expected b/tests/functional/repl/doc-lambda-flavors.expected index 268cac05e..ab5c95639 100644 --- a/tests/functional/repl/doc-lambda-flavors.expected +++ b/tests/functional/repl/doc-lambda-flavors.expected @@ -27,5 +27,3 @@ Function `strictPost`\ … defined at /path/to/tests/functional/repl/doc-comments.nix:41:53 Here's another way to do this - -nix-repl> diff --git a/tests/functional/repl/doc-measurement.expected b/tests/functional/repl/doc-measurement.expected index 14ae11e80..555cac9a2 100644 --- a/tests/functional/repl/doc-measurement.expected +++ b/tests/functional/repl/doc-measurement.expected @@ -9,5 +9,3 @@ Function `measurement`\ … defined at /path/to/tests/functional/repl/doc-comments.nix:13:17 👈 precisely this wide 👉 - -nix-repl> diff --git a/tests/functional/repl/doc-multiply.expected b/tests/functional/repl/doc-multiply.expected index 5cefda516..21523e24c 100644 --- a/tests/functional/repl/doc-multiply.expected +++ b/tests/functional/repl/doc-multiply.expected @@ -15,5 +15,3 @@ Perform *arithmetic* multiplication. It's kind of like repeated **addition**, ve multiply 2 3 => 6 ``` - -nix-repl> diff --git a/tests/functional/repl/doc-unambiguous.expected b/tests/functional/repl/doc-unambiguous.expected index 0872750a1..0db5505d7 100644 --- a/tests/functional/repl/doc-unambiguous.expected +++ b/tests/functional/repl/doc-unambiguous.expected @@ -9,5 +9,3 @@ Function `unambiguous`\ … defined at /path/to/tests/functional/repl/doc-comments.nix:24:5 Very close - -nix-repl> diff --git a/tests/functional/repl/pretty-print-idempotent.expected b/tests/functional/repl/pretty-print-idempotent.expected index f09aec88d..311855dae 100644 --- a/tests/functional/repl/pretty-print-idempotent.expected +++ b/tests/functional/repl/pretty-print-idempotent.expected @@ -35,5 +35,3 @@ nix-repl> twoDeepList [ [ ... ] ] - -nix-repl> From 6e680a664421927cd7d99f956a1e8c4b5bdc7db6 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 24 Jul 2024 01:01:14 +0200 Subject: [PATCH 018/142] tests/functional/repl: Improve failure reporting --- tests/functional/repl.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index a149916ac..706e0f5db 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -311,7 +311,10 @@ for test in $(cd "$testDir/repl"; echo *.in); do in="$testDir/repl/$test.in" actual="$testDir/repl/$test.actual" expected="$testDir/repl/$test.expected" - (cd "$testDir/repl"; set +x; runRepl 2>&1) < "$in" > "$actual" + (cd "$testDir/repl"; set +x; runRepl 2>&1) < "$in" > "$actual" || { + echo "FAIL: $test (exit code $?)" >&2 + badExitCode=1 + } diffAndAcceptInner "$test" "$actual" "$expected" done From 7d4d34a27d8d95e8a63d3b3940bd0ee7b623ff82 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 24 Jul 2024 01:02:21 +0200 Subject: [PATCH 019/142] eval-gc.cc: Fix warning --- src/libexpr/eval-gc.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libexpr/eval-gc.cc b/src/libexpr/eval-gc.cc index 73ab809e5..1bc8cd28f 100644 --- a/src/libexpr/eval-gc.cc +++ b/src/libexpr/eval-gc.cc @@ -84,7 +84,9 @@ void fixupBoehmStackPointer(void ** sp_ptr, void * _pthread_id) { void *& sp = *sp_ptr; auto pthread_id = reinterpret_cast(_pthread_id); +# ifndef __APPLE__ pthread_attr_t pattr; +# endif size_t osStackSize; void * osStackLow; void * osStackBase; From 0bd2d363750db35926460cbc4c3e62dcb1d09bf5 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 24 Jul 2024 12:53:37 +0200 Subject: [PATCH 020/142] Document renderMarkdownToTerminal --- src/libcmd/markdown.hh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/libcmd/markdown.hh b/src/libcmd/markdown.hh index a04d32a4f..164f99b98 100644 --- a/src/libcmd/markdown.hh +++ b/src/libcmd/markdown.hh @@ -5,6 +5,13 @@ namespace nix { +/** + * Render the given Markdown text to the terminal. + * + * If Nix is compiled without Markdown support, this function will return the input text as-is. + * + * The renderer takes into account the terminal width, and wraps text accordingly. + */ std::string renderMarkdownToTerminal(std::string_view markdown); } From e48e0cbab0eed9b9174c27d3f3ddfa1afff5186b Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 24 Jul 2024 12:54:40 +0200 Subject: [PATCH 021/142] markdown.hh: Improve includes --- src/libcmd/markdown.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcmd/markdown.hh b/src/libcmd/markdown.hh index 164f99b98..66db1736c 100644 --- a/src/libcmd/markdown.hh +++ b/src/libcmd/markdown.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "types.hh" +#include namespace nix { From 907b0a371abe92b3314612faea0b3d68e048c156 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 23 Jul 2024 16:02:40 +0200 Subject: [PATCH 022/142] Rename osStackLow -> osStackLimit This is in accordance with ARM's naming convention. "Low" is confusing, because it could refer to either the cold end of the stack as an abstract data type, or a low address. These are different places, because the stack grows down through the address space. --- src/libexpr/eval-gc.cc | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/libexpr/eval-gc.cc b/src/libexpr/eval-gc.cc index 73ab809e5..75368c5ac 100644 --- a/src/libexpr/eval-gc.cc +++ b/src/libexpr/eval-gc.cc @@ -86,12 +86,13 @@ void fixupBoehmStackPointer(void ** sp_ptr, void * _pthread_id) auto pthread_id = reinterpret_cast(_pthread_id); pthread_attr_t pattr; size_t osStackSize; - void * osStackLow; + // The low address of the stack, which grows down. + void * osStackLimit; void * osStackBase; # ifdef __APPLE__ osStackSize = pthread_get_stacksize_np(pthread_id); - osStackLow = pthread_get_stackaddr_np(pthread_id); + osStackLimit = pthread_get_stackaddr_np(pthread_id); # else if (pthread_attr_init(&pattr)) { throw Error("fixupBoehmStackPointer: pthread_attr_init failed"); @@ -110,18 +111,18 @@ void fixupBoehmStackPointer(void ** sp_ptr, void * _pthread_id) # else # error "Need one of `pthread_attr_get_np` or `pthread_getattr_np`" # endif - if (pthread_attr_getstack(&pattr, &osStackLow, &osStackSize)) { + if (pthread_attr_getstack(&pattr, &osStackLimit, &osStackSize)) { throw Error("fixupBoehmStackPointer: pthread_attr_getstack failed"); } if (pthread_attr_destroy(&pattr)) { throw Error("fixupBoehmStackPointer: pthread_attr_destroy failed"); } # endif - osStackBase = (char *) osStackLow + osStackSize; + osStackBase = (char *) osStackLimit + osStackSize; // NOTE: We assume the stack grows down, as it does on all architectures we support. // Architectures that grow the stack up are rare. - if (sp >= osStackBase || sp < osStackLow) { // lo is outside the os stack - sp = osStackLow; + if (sp >= osStackBase || sp < osStackLimit) { // lo is outside the os stack + sp = osStackLimit; } } From 68693276f9c5829489132c1f5a142717da84f44a Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 23 Jul 2024 16:27:26 +0200 Subject: [PATCH 023/142] Update fixupBoehmStackPointer doc lo might have made sense in the bdwgc code, maybe?, but not here. --- src/libexpr/eval-gc.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/eval-gc.cc b/src/libexpr/eval-gc.cc index 75368c5ac..ba19cd74e 100644 --- a/src/libexpr/eval-gc.cc +++ b/src/libexpr/eval-gc.cc @@ -121,7 +121,7 @@ void fixupBoehmStackPointer(void ** sp_ptr, void * _pthread_id) osStackBase = (char *) osStackLimit + osStackSize; // NOTE: We assume the stack grows down, as it does on all architectures we support. // Architectures that grow the stack up are rare. - if (sp >= osStackBase || sp < osStackLimit) { // lo is outside the os stack + if (sp >= osStackBase || sp < osStackLimit) { // sp is outside the os stack sp = osStackLimit; } } From a16df88252a34160fabe7c2710ba11ca51d9fee1 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 24 Jul 2024 15:18:50 +0200 Subject: [PATCH 024/142] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'flake-compat': 'github:edolstra/flake-compat/35bb57c0c8d8b62bbfd284272c928ceb64ddbde9?narHash=sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm%2B504Ch3sNKLd8%3D' (2023-01-17) → 'github:edolstra/flake-compat/0f9255e01c2351cc7d116c072cb317785dd33b33?narHash=sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U%3D' (2023-10-04) • Updated input 'flake-parts': 'github:hercules-ci/flake-parts/9126214d0a59633752a136528f5f3b9aa8565b7d?narHash=sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm%2BGpZNw%3D' (2024-04-01) → 'github:hercules-ci/flake-parts/9227223f6d922fee3c7b190b2cc238a99527bbb7?narHash=sha256-pQMhCCHyQGRzdfAkdJ4cIWiw%2BJNuWsTX7f0ZYSyz0VY%3D' (2024-07-03) • Updated input 'libgit2': 'github:libgit2/libgit2/45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5?narHash=sha256-oX4Z3S9WtJlwvj0uH9HlYcWv%2Bx1hqp8mhXl7HsLu2f0%3D' (2023-10-18) → 'github:libgit2/libgit2/503b66cf00ad7dca940148529f60b1a409ccc462?narHash=sha256-tDUQi%2Bs8sxJ30SmUH7Ln9WmDz5jGatlgKumjwi7KnCo%3D' (2024-07-17) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/be3ca229c85e978880babdeda9748b14e6aa008f?narHash=sha256-L61BXz7n/yNzOeZ3FqlnUmxj4145JOVeq9fvQTQzbNM%3D' (2024-07-21) → 'github:NixOS/nixpkgs/d0907b75146a0ccc1ec0d6c3db287ec287588ef6?narHash=sha256-PhmkdTJs2SfqKzSyDB74rDKp1MH4mGk0pG/%2BWqrnGEw%3D' (2024-07-24) • Updated input 'pre-commit-hooks': 'github:cachix/pre-commit-hooks.nix/40e6053ecb65fcbf12863338a6dcefb3f55f1bf8?narHash=sha256-nMirxrGteNAl9sWiOhoN5tIHyjBbVi5e2tgZUgZlK3Y%3D' (2024-04-12) → 'github:cachix/pre-commit-hooks.nix/f451c19376071a90d8c58ab1a953c6e9840527fd?narHash=sha256-6FPUl7HVtvRHCCBQne7Ylp4p%2BdpP3P/OYuzjztZ4s70%3D' (2024-07-15) • Removed input 'pre-commit-hooks/flake-utils' --- flake.lock | 46 +++++++++++++++------------------------------- 1 file changed, 15 insertions(+), 31 deletions(-) diff --git a/flake.lock b/flake.lock index 1d59439da..932c5fcf5 100644 --- a/flake.lock +++ b/flake.lock @@ -3,11 +3,11 @@ "flake-compat": { "flake": false, "locked": { - "lastModified": 1673956053, - "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", "owner": "edolstra", "repo": "flake-compat", - "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", "type": "github" }, "original": { @@ -23,11 +23,11 @@ ] }, "locked": { - "lastModified": 1712014858, - "narHash": "sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm+GpZNw=", + "lastModified": 1719994518, + "narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "9126214d0a59633752a136528f5f3b9aa8565b7d", + "rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7", "type": "github" }, "original": { @@ -36,29 +36,14 @@ "type": "github" } }, - "flake-utils": { - "locked": { - "lastModified": 1667395993, - "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, "libgit2": { "flake": false, "locked": { - "lastModified": 1697646580, - "narHash": "sha256-oX4Z3S9WtJlwvj0uH9HlYcWv+x1hqp8mhXl7HsLu2f0=", + "lastModified": 1721209236, + "narHash": "sha256-tDUQi+s8sxJ30SmUH7Ln9WmDz5jGatlgKumjwi7KnCo=", "owner": "libgit2", "repo": "libgit2", - "rev": "45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5", + "rev": "503b66cf00ad7dca940148529f60b1a409ccc462", "type": "github" }, "original": { @@ -69,11 +54,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1721560568, - "narHash": "sha256-L61BXz7n/yNzOeZ3FqlnUmxj4145JOVeq9fvQTQzbNM=", + "lastModified": 1721821769, + "narHash": "sha256-PhmkdTJs2SfqKzSyDB74rDKp1MH4mGk0pG/+WqrnGEw=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "be3ca229c85e978880babdeda9748b14e6aa008f", + "rev": "d0907b75146a0ccc1ec0d6c3db287ec287588ef6", "type": "github" }, "original": { @@ -118,7 +103,6 @@ "pre-commit-hooks": { "inputs": { "flake-compat": [], - "flake-utils": "flake-utils", "gitignore": [], "nixpkgs": [ "nixpkgs" @@ -128,11 +112,11 @@ ] }, "locked": { - "lastModified": 1712897695, - "narHash": "sha256-nMirxrGteNAl9sWiOhoN5tIHyjBbVi5e2tgZUgZlK3Y=", + "lastModified": 1721042469, + "narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=", "owner": "cachix", "repo": "pre-commit-hooks.nix", - "rev": "40e6053ecb65fcbf12863338a6dcefb3f55f1bf8", + "rev": "f451c19376071a90d8c58ab1a953c6e9840527fd", "type": "github" }, "original": { From 650f1894020c586360f30fa4e728fe9eea76d114 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 24 Jul 2024 15:24:10 +0200 Subject: [PATCH 025/142] flake.nix: Pin libgit2 to a release --- flake.lock | 7 ++++--- flake.nix | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/flake.lock b/flake.lock index 932c5fcf5..96c35b088 100644 --- a/flake.lock +++ b/flake.lock @@ -39,15 +39,16 @@ "libgit2": { "flake": false, "locked": { - "lastModified": 1721209236, - "narHash": "sha256-tDUQi+s8sxJ30SmUH7Ln9WmDz5jGatlgKumjwi7KnCo=", + "lastModified": 1715853528, + "narHash": "sha256-J2rCxTecyLbbDdsyBWn9w7r3pbKRMkI9E7RvRgAqBdY=", "owner": "libgit2", "repo": "libgit2", - "rev": "503b66cf00ad7dca940148529f60b1a409ccc462", + "rev": "36f7e21ad757a3dacc58cf7944329da6bc1d6e96", "type": "github" }, "original": { "owner": "libgit2", + "ref": "v1.8.1", "repo": "libgit2", "type": "github" } diff --git a/flake.nix b/flake.nix index ff2c8ecfa..45c493c6f 100644 --- a/flake.nix +++ b/flake.nix @@ -7,7 +7,7 @@ inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; - inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; }; + inputs.libgit2 = { url = "github:libgit2/libgit2/v1.8.1"; flake = false; }; # dev tooling inputs.flake-parts.url = "github:hercules-ci/flake-parts"; From f9a23c8d2180e644afe930c07680eba9e360d9c5 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 24 Jul 2024 15:30:28 +0200 Subject: [PATCH 026/142] flake.nix: Remove outdated comment --- flake.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/flake.nix b/flake.nix index 45c493c6f..bfa4c409a 100644 --- a/flake.nix +++ b/flake.nix @@ -1,8 +1,6 @@ { description = "The purely functional package manager"; - # TODO switch to nixos-23.11-small - # https://nixpk.gs/pr-tracker.html?pr=291954 inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-24.05"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; From c316f1557d9c71a0a6864fa6774a913a52748023 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 24 Jul 2024 15:30:56 +0200 Subject: [PATCH 027/142] flake: Switch to nixos-24.05 channel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/d0907b75146a0ccc1ec0d6c3db287ec287588ef6?narHash=sha256-PhmkdTJs2SfqKzSyDB74rDKp1MH4mGk0pG/%2BWqrnGEw%3D' (2024-07-24) → 'github:NixOS/nixpkgs/63d37ccd2d178d54e7fb691d7ec76000740ea24a?narHash=sha256-7cCC8%2BTdq1%2B3OPyc3%2BgVo9dzUNkNIQfwSDJ2HSi2u3o%3D' (2024-07-21) --- flake.lock | 8 ++++---- flake.nix | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index 96c35b088..d1b54eb87 100644 --- a/flake.lock +++ b/flake.lock @@ -55,16 +55,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1721821769, - "narHash": "sha256-PhmkdTJs2SfqKzSyDB74rDKp1MH4mGk0pG/+WqrnGEw=", + "lastModified": 1721548954, + "narHash": "sha256-7cCC8+Tdq1+3OPyc3+gVo9dzUNkNIQfwSDJ2HSi2u3o=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "d0907b75146a0ccc1ec0d6c3db287ec287588ef6", + "rev": "63d37ccd2d178d54e7fb691d7ec76000740ea24a", "type": "github" }, "original": { "owner": "NixOS", - "ref": "release-24.05", + "ref": "nixos-24.05", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index bfa4c409a..e6af87723 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,7 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-24.05"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; From 171ef75218ec1bc0cdfe7b1b49796452d1d6bea6 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 24 Jul 2024 15:55:57 +0200 Subject: [PATCH 028/142] Rename pre-commit-hooks -> git-hooks-nix Following the upstream rename --- flake.lock | 54 ++++++++++++++++++------------------ flake.nix | 10 +++---- maintainers/flake-module.nix | 2 +- 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/flake.lock b/flake.lock index d1b54eb87..2ac413a69 100644 --- a/flake.lock +++ b/flake.lock @@ -36,6 +36,31 @@ "type": "github" } }, + "git-hooks-nix": { + "inputs": { + "flake-compat": [], + "gitignore": [], + "nixpkgs": [ + "nixpkgs" + ], + "nixpkgs-stable": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1721042469, + "narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=", + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "f451c19376071a90d8c58ab1a953c6e9840527fd", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "git-hooks.nix", + "type": "github" + } + }, "libgit2": { "flake": false, "locked": { @@ -101,40 +126,15 @@ "type": "github" } }, - "pre-commit-hooks": { - "inputs": { - "flake-compat": [], - "gitignore": [], - "nixpkgs": [ - "nixpkgs" - ], - "nixpkgs-stable": [ - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1721042469, - "narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=", - "owner": "cachix", - "repo": "pre-commit-hooks.nix", - "rev": "f451c19376071a90d8c58ab1a953c6e9840527fd", - "type": "github" - }, - "original": { - "owner": "cachix", - "repo": "pre-commit-hooks.nix", - "type": "github" - } - }, "root": { "inputs": { "flake-compat": "flake-compat", "flake-parts": "flake-parts", + "git-hooks-nix": "git-hooks-nix", "libgit2": "libgit2", "nixpkgs": "nixpkgs", "nixpkgs-23-11": "nixpkgs-23-11", - "nixpkgs-regression": "nixpkgs-regression", - "pre-commit-hooks": "pre-commit-hooks" + "nixpkgs-regression": "nixpkgs-regression" } } }, diff --git a/flake.nix b/flake.nix index e6af87723..2384c2974 100644 --- a/flake.nix +++ b/flake.nix @@ -9,14 +9,14 @@ # dev tooling inputs.flake-parts.url = "github:hercules-ci/flake-parts"; - inputs.pre-commit-hooks.url = "github:cachix/pre-commit-hooks.nix"; + inputs.git-hooks-nix.url = "github:cachix/git-hooks.nix"; # work around https://github.com/NixOS/nix/issues/7730 inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; - inputs.pre-commit-hooks.inputs.nixpkgs.follows = "nixpkgs"; - inputs.pre-commit-hooks.inputs.nixpkgs-stable.follows = "nixpkgs"; + inputs.git-hooks-nix.inputs.nixpkgs.follows = "nixpkgs"; + inputs.git-hooks-nix.inputs.nixpkgs-stable.follows = "nixpkgs"; # work around 7730 and https://github.com/NixOS/nix/issues/7807 - inputs.pre-commit-hooks.inputs.flake-compat.follows = ""; - inputs.pre-commit-hooks.inputs.gitignore.follows = ""; + inputs.git-hooks-nix.inputs.flake-compat.follows = ""; + inputs.git-hooks-nix.inputs.gitignore.follows = ""; outputs = inputs@{ self, nixpkgs, nixpkgs-regression, libgit2, ... }: diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index b5c7bfd53..be91df536 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -2,7 +2,7 @@ { imports = [ - inputs.pre-commit-hooks.flakeModule + inputs.git-hooks-nix.flakeModule ]; perSystem = { config, pkgs, ... }: { From 3be7c0037eb4728e201b49572230d563cd2ac096 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 19 Jul 2024 15:48:19 +0200 Subject: [PATCH 029/142] WorkerProto: Support fine-grained protocol feature negotiation Currently, the worker protocol has a version number that we increment whenever we change something in the protocol. However, this can cause a collision between Nix PRs / forks that make protocol changes (e.g. PR #9857 increments the version, which could collide with another PR). So instead, the client and daemon now exchange a set of protocol features (such as `auth-forwarding`). They will use the intersection of the sets of features, i.e. the features they both support. Note that protocol features are completely distinct from `ExperimentalFeature`s. --- src/libstore/daemon.cc | 11 ++--- src/libstore/remote-store.cc | 10 ++++- src/libstore/worker-protocol-connection.cc | 51 +++++++++++++++++++--- src/libstore/worker-protocol-connection.hh | 27 ++++++++++-- src/libstore/worker-protocol.hh | 8 +++- tests/unit/libstore/worker-protocol.cc | 49 ++++++++++++++++----- 6 files changed, 127 insertions(+), 29 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 6533b2f58..94f00cfb6 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -1025,19 +1025,20 @@ void processConnection( #endif /* Exchange the greeting. */ - WorkerProto::Version clientVersion = + auto [protoVersion, features] = WorkerProto::BasicServerConnection::handshake( - to, from, PROTOCOL_VERSION); + to, from, PROTOCOL_VERSION, WorkerProto::allFeatures); - if (clientVersion < 0x10a) + if (protoVersion < 0x10a) throw Error("the Nix client version is too old"); WorkerProto::BasicServerConnection conn; conn.to = std::move(to); conn.from = std::move(from); - conn.protoVersion = clientVersion; + conn.protoVersion = protoVersion; + conn.features = features; - auto tunnelLogger = new TunnelLogger(conn.to, clientVersion); + auto tunnelLogger = new TunnelLogger(conn.to, protoVersion); auto prevLogger = nix::logger; // FIXME if (!recursive) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index ebb0864c5..555936c18 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -73,8 +73,11 @@ void RemoteStore::initConnection(Connection & conn) StringSink saved; TeeSource tee(conn.from, saved); try { - conn.protoVersion = WorkerProto::BasicClientConnection::handshake( - conn.to, tee, PROTOCOL_VERSION); + auto [protoVersion, features] = WorkerProto::BasicClientConnection::handshake( + conn.to, tee, PROTOCOL_VERSION, + WorkerProto::allFeatures); + conn.protoVersion = protoVersion; + conn.features = features; } catch (SerialisationError & e) { /* In case the other side is waiting for our input, close it. */ @@ -88,6 +91,9 @@ void RemoteStore::initConnection(Connection & conn) static_cast(conn) = conn.postHandshake(*this); + for (auto & feature : conn.features) + debug("negotiated feature '%s'", feature); + auto ex = conn.processStderrReturn(); if (ex) std::rethrow_exception(ex); } diff --git a/src/libstore/worker-protocol-connection.cc b/src/libstore/worker-protocol-connection.cc index 93d13d48e..a47dbb689 100644 --- a/src/libstore/worker-protocol-connection.cc +++ b/src/libstore/worker-protocol-connection.cc @@ -5,6 +5,8 @@ namespace nix { +const std::set WorkerProto::allFeatures{}; + WorkerProto::BasicClientConnection::~BasicClientConnection() { try { @@ -137,8 +139,21 @@ void WorkerProto::BasicClientConnection::processStderr(bool * daemonException, S } } -WorkerProto::Version -WorkerProto::BasicClientConnection::handshake(BufferedSink & to, Source & from, WorkerProto::Version localVersion) +static std::set +intersectFeatures(const std::set & a, const std::set & b) +{ + std::set res; + for (auto & x : a) + if (b.contains(x)) + res.insert(x); + return res; +} + +std::tuple> WorkerProto::BasicClientConnection::handshake( + BufferedSink & to, + Source & from, + WorkerProto::Version localVersion, + const std::set & supportedFeatures) { to << WORKER_MAGIC_1 << localVersion; to.flush(); @@ -153,11 +168,24 @@ WorkerProto::BasicClientConnection::handshake(BufferedSink & to, Source & from, if (GET_PROTOCOL_MINOR(daemonVersion) < 10) throw Error("the Nix daemon version is too old"); - return std::min(daemonVersion, localVersion); + auto protoVersion = std::min(daemonVersion, localVersion); + + /* Exchange features. */ + std::set daemonFeatures; + if (GET_PROTOCOL_MINOR(protoVersion) >= 38) { + to << supportedFeatures; + to.flush(); + daemonFeatures = readStrings>(from); + } + + return {protoVersion, intersectFeatures(daemonFeatures, supportedFeatures)}; } -WorkerProto::Version -WorkerProto::BasicServerConnection::handshake(BufferedSink & to, Source & from, WorkerProto::Version localVersion) +std::tuple> WorkerProto::BasicServerConnection::handshake( + BufferedSink & to, + Source & from, + WorkerProto::Version localVersion, + const std::set & supportedFeatures) { unsigned int magic = readInt(from); if (magic != WORKER_MAGIC_1) @@ -165,7 +193,18 @@ WorkerProto::BasicServerConnection::handshake(BufferedSink & to, Source & from, to << WORKER_MAGIC_2 << localVersion; to.flush(); auto clientVersion = readInt(from); - return std::min(clientVersion, localVersion); + + auto protoVersion = std::min(clientVersion, localVersion); + + /* Exchange features. */ + std::set clientFeatures; + if (GET_PROTOCOL_MINOR(protoVersion) >= 38) { + clientFeatures = readStrings>(from); + to << supportedFeatures; + to.flush(); + } + + return {protoVersion, intersectFeatures(clientFeatures, supportedFeatures)}; } WorkerProto::ClientHandshakeInfo WorkerProto::BasicClientConnection::postHandshake(const StoreDirConfig & store) diff --git a/src/libstore/worker-protocol-connection.hh b/src/libstore/worker-protocol-connection.hh index 38287d08e..9c96195b5 100644 --- a/src/libstore/worker-protocol-connection.hh +++ b/src/libstore/worker-protocol-connection.hh @@ -23,6 +23,11 @@ struct WorkerProto::BasicConnection */ WorkerProto::Version protoVersion; + /** + * The set of features that both sides support. + */ + std::set features; + /** * Coercion to `WorkerProto::ReadConn`. This makes it easy to use the * factored out serve protocol serializers with a @@ -72,8 +77,8 @@ struct WorkerProto::BasicClientConnection : WorkerProto::BasicConnection /** * Establishes connection, negotiating version. * - * @return the version provided by the other side of the - * connection. + * @return the minimum version supported by both sides and the set + * of protocol features supported by both sides. * * @param to Taken by reference to allow for various error handling * mechanisms. @@ -82,8 +87,15 @@ struct WorkerProto::BasicClientConnection : WorkerProto::BasicConnection * handling mechanisms. * * @param localVersion Our version which is sent over + * + * @param features The protocol features that we support */ - static Version handshake(BufferedSink & to, Source & from, WorkerProto::Version localVersion); + // FIXME: this should probably be a constructor. + static std::tuple> handshake( + BufferedSink & to, + Source & from, + WorkerProto::Version localVersion, + const std::set & supportedFeatures); /** * After calling handshake, must call this to exchange some basic @@ -138,8 +150,15 @@ struct WorkerProto::BasicServerConnection : WorkerProto::BasicConnection * handling mechanisms. * * @param localVersion Our version which is sent over + * + * @param features The protocol features that we support */ - static WorkerProto::Version handshake(BufferedSink & to, Source & from, WorkerProto::Version localVersion); + // FIXME: this should probably be a constructor. + static std::tuple> handshake( + BufferedSink & to, + Source & from, + WorkerProto::Version localVersion, + const std::set & supportedFeatures); /** * After calling handshake, must call this to exchange some basic diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 9fc16d015..c356fa1bf 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -11,7 +11,9 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION (1 << 8 | 37) +/* Note: you generally shouldn't change the protocol version. Define a + new `WorkerProto::Feature` instead. */ +#define PROTOCOL_VERSION (1 << 8 | 38) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -131,6 +133,10 @@ struct WorkerProto { WorkerProto::Serialise::write(store, conn, t); } + + using Feature = std::string; + + static const std::set allFeatures; }; enum struct WorkerProto::Op : uint64_t diff --git a/tests/unit/libstore/worker-protocol.cc b/tests/unit/libstore/worker-protocol.cc index c15120010..bbea9ed75 100644 --- a/tests/unit/libstore/worker-protocol.cc +++ b/tests/unit/libstore/worker-protocol.cc @@ -658,15 +658,15 @@ TEST_F(WorkerProtoTest, handshake_log) FdSink out { toServer.writeSide.get() }; FdSource in0 { toClient.readSide.get() }; TeeSource in { in0, toClientLog }; - clientResult = WorkerProto::BasicClientConnection::handshake( - out, in, defaultVersion); + clientResult = std::get<0>(WorkerProto::BasicClientConnection::handshake( + out, in, defaultVersion, {})); }); { FdSink out { toClient.writeSide.get() }; FdSource in { toServer.readSide.get() }; WorkerProto::BasicServerConnection::handshake( - out, in, defaultVersion); + out, in, defaultVersion, {}); }; thread.join(); @@ -675,6 +675,33 @@ TEST_F(WorkerProtoTest, handshake_log) }); } +TEST_F(WorkerProtoTest, handshake_features) +{ + Pipe toClient, toServer; + toClient.create(); + toServer.create(); + + std::tuple> clientResult; + + auto clientThread = std::thread([&]() { + FdSink out { toServer.writeSide.get() }; + FdSource in { toClient.readSide.get() }; + clientResult = WorkerProto::BasicClientConnection::handshake( + out, in, 123, {"bar", "aap", "mies", "xyzzy"}); + }); + + FdSink out { toClient.writeSide.get() }; + FdSource in { toServer.readSide.get() }; + auto daemonResult = WorkerProto::BasicServerConnection::handshake( + out, in, 456, {"foo", "bar", "xyzzy"}); + + clientThread.join(); + + EXPECT_EQ(clientResult, daemonResult); + EXPECT_EQ(std::get<0>(clientResult), 123); + EXPECT_EQ(std::get<1>(clientResult), std::set({"bar", "xyzzy"})); +} + /// Has to be a `BufferedSink` for handshake. struct NullBufferedSink : BufferedSink { void writeUnbuffered(std::string_view data) override { } @@ -686,8 +713,8 @@ TEST_F(WorkerProtoTest, handshake_client_replay) NullBufferedSink nullSink; StringSource in { toClientLog }; - auto clientResult = WorkerProto::BasicClientConnection::handshake( - nullSink, in, defaultVersion); + auto clientResult = std::get<0>(WorkerProto::BasicClientConnection::handshake( + nullSink, in, defaultVersion, {})); EXPECT_EQ(clientResult, defaultVersion); }); @@ -705,13 +732,13 @@ TEST_F(WorkerProtoTest, handshake_client_truncated_replay_throws) if (len < 8) { EXPECT_THROW( WorkerProto::BasicClientConnection::handshake( - nullSink, in, defaultVersion), + nullSink, in, defaultVersion, {}), EndOfFile); } else { // Not sure why cannot keep on checking for `EndOfFile`. EXPECT_THROW( WorkerProto::BasicClientConnection::handshake( - nullSink, in, defaultVersion), + nullSink, in, defaultVersion, {}), Error); } } @@ -734,17 +761,17 @@ TEST_F(WorkerProtoTest, handshake_client_corrupted_throws) // magic bytes don't match EXPECT_THROW( WorkerProto::BasicClientConnection::handshake( - nullSink, in, defaultVersion), + nullSink, in, defaultVersion, {}), Error); } else if (idx < 8 || idx >= 12) { // Number out of bounds EXPECT_THROW( WorkerProto::BasicClientConnection::handshake( - nullSink, in, defaultVersion), + nullSink, in, defaultVersion, {}), SerialisationError); } else { - auto ver = WorkerProto::BasicClientConnection::handshake( - nullSink, in, defaultVersion); + auto ver = std::get<0>(WorkerProto::BasicClientConnection::handshake( + nullSink, in, defaultVersion, {})); // `std::min` of this and the other version saves us EXPECT_EQ(ver, defaultVersion); } From 3172e88af544ca53a54930ccf3f580b15141f01c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 20 Jul 2024 22:46:09 +0200 Subject: [PATCH 030/142] Make abort() call sites log first --- src/libcmd/repl.cc | 2 +- src/libexpr/attr-path.cc | 2 +- src/libexpr/eval.cc | 8 +++---- src/libexpr/nixexpr.cc | 4 ++-- src/libexpr/primops.cc | 2 +- src/libexpr/print-ambiguous.cc | 2 +- src/libexpr/print.cc | 4 ++-- src/libexpr/symbol-table.hh | 3 ++- src/libexpr/value.hh | 2 +- src/libfetchers/attrs.cc | 4 ++-- src/libmain/loggers.cc | 2 +- .../build/drv-output-substitution-goal.hh | 2 +- src/libstore/build/goal.hh | 4 ++-- src/libstore/build/substitution-goal.hh | 2 +- src/libstore/build/worker.cc | 4 ++-- src/libstore/daemon.cc | 2 +- src/libstore/gc.cc | 2 +- src/libstore/globals.cc | 2 +- src/libstore/nar-info-disk-cache.cc | 4 ++-- .../unix/build/local-derivation-goal.cc | 2 +- src/libstore/unix/pathlocks.cc | 2 +- src/libutil/chunked-vector.hh | 4 +++- src/libutil/error.cc | 12 ++++++++++ src/libutil/error.hh | 24 +++++++++++++++++++ src/libutil/file-content-address.cc | 2 +- src/libutil/fs-sink.cc | 2 +- src/libutil/git.cc | 2 +- src/libutil/hash.cc | 2 +- src/libutil/logging.cc | 2 +- src/libutil/serialise.cc | 8 +++---- src/libutil/sync.hh | 4 +++- src/libutil/unix/monitor-fd.hh | 4 +++- src/libutil/unix/processes.cc | 2 +- src/nix-env/nix-env.cc | 2 +- src/nix-store/nix-store.cc | 2 +- 35 files changed, 88 insertions(+), 45 deletions(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index b5d0816dd..bf0d820c2 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -217,7 +217,7 @@ ReplExitStatus NixRepl::mainLoop() case ProcessLineResult::PromptAgain: break; default: - abort(); + unreachable(); } } catch (ParseError & e) { if (e.msg().find("unexpected end of file") != std::string::npos) { diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index d61d93630..2f67260c5 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -134,7 +134,7 @@ std::pair findPackageFilename(EvalState & state, Value & v return {SourcePath{path.accessor, CanonPath(fn.substr(0, colon))}, lineno}; } catch (std::invalid_argument & e) { fail(); - abort(); + unreachable(); } } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index b192f9b4b..32bc68e6d 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -149,7 +149,7 @@ std::string_view showType(ValueType type, bool withArticle) case nFloat: return WA("a", "float"); case nThunk: return WA("a", "thunk"); } - abort(); + unreachable(); } @@ -771,7 +771,7 @@ void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr & case ReplExitStatus::Continue: break; default: - abort(); + unreachable(); } } } @@ -1140,7 +1140,7 @@ inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v, const PosIdx po void Expr::eval(EvalState & state, Env & env, Value & v) { - abort(); + unreachable(); } @@ -1573,7 +1573,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & .withFrame(*fun.payload.lambda.env, lambda) .debugThrow(); } - abort(); // can't happen + unreachable(); } } diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 6c6769cfd..dbc74faf9 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -25,7 +25,7 @@ std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol) void Expr::show(const SymbolTable & symbols, std::ostream & str) const { - abort(); + unreachable(); } void ExprInt::show(const SymbolTable & symbols, std::ostream & str) const @@ -271,7 +271,7 @@ std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath) void Expr::bindVars(EvalState & es, const std::shared_ptr & env) { - abort(); + unreachable(); } void ExprInt::bindVars(EvalState & es, const std::shared_ptr & env) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5a373a43b..0b3b19b57 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -426,7 +426,7 @@ static void prim_typeOf(EvalState & state, const PosIdx pos, Value * * args, Val t = args[0]->external()->typeOf(); break; case nFloat: t = "float"; break; - case nThunk: abort(); + case nThunk: unreachable(); } v.mkString(t); } diff --git a/src/libexpr/print-ambiguous.cc b/src/libexpr/print-ambiguous.cc index 5d55b45da..a40c98643 100644 --- a/src/libexpr/print-ambiguous.cc +++ b/src/libexpr/print-ambiguous.cc @@ -94,7 +94,7 @@ void printAmbiguous( break; default: printError("Nix evaluator internal error: printAmbiguous: invalid value type"); - abort(); + unreachable(); } } diff --git a/src/libexpr/print.cc b/src/libexpr/print.cc index bc17d6bfe..4d1a6868c 100644 --- a/src/libexpr/print.cc +++ b/src/libexpr/print.cc @@ -475,7 +475,7 @@ private: else output << "primop"; } else { - abort(); + unreachable(); } output << "»"; @@ -504,7 +504,7 @@ private: if (options.ansiColors) output << ANSI_NORMAL; } else { - abort(); + unreachable(); } } diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh index c7a3563b0..dee7369e8 100644 --- a/src/libexpr/symbol-table.hh +++ b/src/libexpr/symbol-table.hh @@ -7,6 +7,7 @@ #include "types.hh" #include "chunked-vector.hh" +#include "error.hh" namespace nix { @@ -113,7 +114,7 @@ public: SymbolStr operator[](Symbol s) const { if (s.id == 0 || s.id > store.size()) - abort(); + unreachable(); return SymbolStr(store[s.id - 1]); } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 1f4d72d39..fdc6c84c4 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -285,7 +285,7 @@ public: if (invalidIsThunk) return nThunk; else - abort(); + unreachable(); } inline void finishValue(InternalType newType, Payload newPayload) diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index b788c5948..25d04cdc9 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -33,7 +33,7 @@ nlohmann::json attrsToJSON(const Attrs & attrs) json[attr.first] = *v; } else if (auto v = std::get_if>(&attr.second)) { json[attr.first] = v->t; - } else abort(); + } else unreachable(); } return json; } @@ -99,7 +99,7 @@ std::map attrsToQuery(const Attrs & attrs) query.insert_or_assign(attr.first, *v); } else if (auto v = std::get_if>(&attr.second)) { query.insert_or_assign(attr.first, v->t ? "1" : "0"); - } else abort(); + } else unreachable(); } return query; } diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc index 9829859de..a4e0530c8 100644 --- a/src/libmain/loggers.cc +++ b/src/libmain/loggers.cc @@ -36,7 +36,7 @@ Logger * makeDefaultLogger() { return logger; } default: - abort(); + unreachable(); } } diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh index 807054926..8c60d0198 100644 --- a/src/libstore/build/drv-output-substitution-goal.hh +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -36,7 +36,7 @@ public: Co init() override; Co realisationFetched(std::shared_ptr outputInfo, nix::ref sub); - void timedOut(Error && ex) override { abort(); }; + void timedOut(Error && ex) override { unreachable(); }; std::string key() override; diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 162c392d0..9c6a40c84 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -400,12 +400,12 @@ public: virtual void handleChildOutput(Descriptor fd, std::string_view data) { - abort(); + unreachable(); } virtual void handleEOF(Descriptor fd) { - abort(); + unreachable(); } void trace(std::string_view s); diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index 86e4f5423..c1de45379 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -50,7 +50,7 @@ public: PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); ~PathSubstitutionGoal(); - void timedOut(Error && ex) override { abort(); }; + void timedOut(Error && ex) override { unreachable(); }; /** * We prepend "a$" to the key name to ensure substitution goals diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 7fc41b121..ab0ba67b5 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -216,7 +216,7 @@ void Worker::childStarted(GoalPtr goal, const std::set std::string BaseSetting::to_string() const if (value == smEnabled) return "true"; else if (value == smRelaxed) return "relaxed"; else if (value == smDisabled) return "false"; - else abort(); + else unreachable(); } template<> void BaseSetting::convertToArg(Args & args, const std::string & category) diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 288f618d5..83e63794e 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -164,7 +164,7 @@ public: Cache & getCache(State & state, const std::string & uri) { auto i = state.caches.find(uri); - if (i == state.caches.end()) abort(); + if (i == state.caches.end()) unreachable(); return i->second; } @@ -211,7 +211,7 @@ public: { auto r(state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority)); - if (!r.next()) { abort(); } + if (!r.next()) { unreachable(); } ret.id = (int) r.getInt(0); } diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 0dd102200..ac5b4dd0b 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -165,7 +165,7 @@ void LocalDerivationGoal::killSandbox(bool getStats) buildResult.cpuSystem = stats.cpuSystem; } #else - abort(); + unreachable(); #endif } diff --git a/src/libstore/unix/pathlocks.cc b/src/libstore/unix/pathlocks.cc index af21319a7..1ec4579ec 100644 --- a/src/libstore/unix/pathlocks.cc +++ b/src/libstore/unix/pathlocks.cc @@ -45,7 +45,7 @@ bool lockFile(Descriptor desc, LockType lockType, bool wait) if (lockType == ltRead) type = LOCK_SH; else if (lockType == ltWrite) type = LOCK_EX; else if (lockType == ltNone) type = LOCK_UN; - else abort(); + else unreachable(); if (wait) { while (flock(desc, type) != 0) { diff --git a/src/libutil/chunked-vector.hh b/src/libutil/chunked-vector.hh index d914e2542..4709679a6 100644 --- a/src/libutil/chunked-vector.hh +++ b/src/libutil/chunked-vector.hh @@ -6,6 +6,8 @@ #include #include +#include "error.hh" + namespace nix { /** @@ -30,7 +32,7 @@ private: auto & addChunk() { if (size_ >= std::numeric_limits::max() - ChunkSize) - abort(); + unreachable(); chunks.emplace_back(); chunks.back().reserve(ChunkSize); return chunks.back(); diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 33c391963..b1858911a 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -430,4 +430,16 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s return out; } +void panic(std::string_view msg) +{ + printError(msg); + printError("This was a fatal error, aborting."); + abort(); +} + +void panic(const char * file, int line, const char * func) +{ + panic(std::string("Unexpected condition in ") + func + " at " + file + ":" + std::to_string(line)); +} + } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index d7fe902d6..572a1baf7 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -273,4 +273,28 @@ using NativeSysError = */ void throwExceptionSelfCheck(); +/** + * Print a message and abort(). + * + * @note: This assumes that the logger is operational + */ +[[noreturn]] +void panic(std::string_view msg); + +/** + * Print a basic error message with source position and abort(). + * Use the unreachable macro to call this. + * + * @note: This assumes that the logger is operational + */ +[[noreturn]] +void panic(const char * file, int line, const char * func); + +/** + * Print a basic error message with source position and abort(). + * + * @note: This assumes that the logger is operational + */ +#define unreachable() (panic(__FILE__, __LINE__, __func__)) + } diff --git a/src/libutil/file-content-address.cc b/src/libutil/file-content-address.cc index 438dac7da..86378dd67 100644 --- a/src/libutil/file-content-address.cc +++ b/src/libutil/file-content-address.cc @@ -63,7 +63,7 @@ std::string_view renderFileIngestionMethod(FileIngestionMethod method) case FileIngestionMethod::Git: return "git"; default: - abort(); + unreachable(); } } diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc index 3246e0902..f15324d0a 100644 --- a/src/libutil/fs-sink.cc +++ b/src/libutil/fs-sink.cc @@ -53,7 +53,7 @@ void copyRecursive( throw Error("file '%1%' has an unsupported type", from); default: - abort(); + unreachable(); } } diff --git a/src/libutil/git.cc b/src/libutil/git.cc index a6968a43e..af91fa643 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -201,7 +201,7 @@ std::optional convertMode(SourceAccessor::Type type) case SourceAccessor::tRegular: return Mode::Regular; case SourceAccessor::tDirectory: return Mode::Directory; case SourceAccessor::tMisc: return std::nullopt; - default: abort(); + default: unreachable(); } } diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 35b913e42..ab2a8695d 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -25,7 +25,7 @@ static size_t regularHashSize(HashAlgorithm type) { case HashAlgorithm::SHA256: return sha256HashSize; case HashAlgorithm::SHA512: return sha512HashSize; } - abort(); + unreachable(); } diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 55751b4cf..29427f2f6 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -189,7 +189,7 @@ struct JSONLogger : Logger { else if (f.type == Logger::Field::tString) arr.push_back(f.s); else - abort(); + unreachable(); } void write(const nlohmann::json & json) diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 36b99905a..ee4351545 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -260,7 +260,7 @@ std::unique_ptr sourceToSink(std::function fun) }); } - if (!*coro) { abort(); } + if (!*coro) { unreachable(); } if (!cur.empty()) { CoroutineContext ctx; @@ -271,12 +271,12 @@ std::unique_ptr sourceToSink(std::function fun) void finish() override { if (!coro) return; - if (!*coro) abort(); + if (!*coro) unreachable(); { CoroutineContext ctx; (*coro)(true); } - if (*coro) abort(); + if (*coro) unreachable(); } }; @@ -316,7 +316,7 @@ std::unique_ptr sinkToSource( }); } - if (!*coro) { eof(); abort(); } + if (!*coro) { eof(); unreachable(); } if (pos == cur.size()) { if (!cur.empty()) { diff --git a/src/libutil/sync.hh b/src/libutil/sync.hh index 20dd6ee52..c1b699ffc 100644 --- a/src/libutil/sync.hh +++ b/src/libutil/sync.hh @@ -7,6 +7,8 @@ #include #include +#include "error.hh" + namespace nix { /** @@ -47,7 +49,7 @@ public: friend SyncBase; Lock(SyncBase * s) : s(s), lk(s->mutex) { } public: - Lock(Lock && l) : s(l.s) { abort(); } + Lock(Lock && l) : s(l.s) { unreachable(); } Lock(const Lock & l) = delete; ~Lock() { } diff --git a/src/libutil/unix/monitor-fd.hh b/src/libutil/unix/monitor-fd.hh index 103894de9..b6610feff 100644 --- a/src/libutil/unix/monitor-fd.hh +++ b/src/libutil/unix/monitor-fd.hh @@ -40,7 +40,9 @@ public: #endif ; auto count = poll(fds, 1, -1); - if (count == -1) abort(); // can't happen + if (count == -1) + unreachable(); + /* This shouldn't happen, but can on macOS due to a bug. See rdar://37550628. diff --git a/src/libutil/unix/processes.cc b/src/libutil/unix/processes.cc index 1af559a21..c5ce74acc 100644 --- a/src/libutil/unix/processes.cc +++ b/src/libutil/unix/processes.cc @@ -182,7 +182,7 @@ static pid_t doFork(bool allowVfork, ChildWrapperFunction & fun) #endif if (pid != 0) return pid; fun(); - abort(); + unreachable(); } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 5e170c99d..40c200542 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1159,7 +1159,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) case cvEqual: ch = '='; break; case cvGreater: ch = '<'; break; case cvUnavail: ch = '-'; break; - default: abort(); + default: unreachable(); } if (xmlOutput) { diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index f073074e8..b4de42ba1 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -480,7 +480,7 @@ static void opQuery(Strings opFlags, Strings opArgs) } default: - abort(); + unreachable(); } } From e086d5d899aaefdce8cc7a509cb2971be9732b2b Mon Sep 17 00:00:00 2001 From: Ryan Hendrickson Date: Wed, 24 Jul 2024 13:17:28 -0400 Subject: [PATCH 031/142] libexpr: experimental pipe operators --- doc/manual/rl-next/pipe-operators.md | 28 ++++++++++ doc/manual/src/language/operators.md | 37 ++++++++++++- src/libexpr/lexer.l | 14 +++++ src/libexpr/parser.y | 30 ++++++++--- src/libutil/experimental-features.cc | 10 +++- src/libutil/experimental-features.hh | 1 + .../lang/eval-fail-pipe-operators.err.exp | 5 ++ .../lang/eval-fail-pipe-operators.nix | 1 + tests/unit/libexpr/main.cc | 3 ++ tests/unit/libexpr/trivial.cc | 54 +++++++++++++++++++ 10 files changed, 174 insertions(+), 9 deletions(-) create mode 100644 doc/manual/rl-next/pipe-operators.md create mode 100644 tests/functional/lang/eval-fail-pipe-operators.err.exp create mode 100644 tests/functional/lang/eval-fail-pipe-operators.nix diff --git a/doc/manual/rl-next/pipe-operators.md b/doc/manual/rl-next/pipe-operators.md new file mode 100644 index 000000000..b4cbe30e3 --- /dev/null +++ b/doc/manual/rl-next/pipe-operators.md @@ -0,0 +1,28 @@ +--- +synopsis: "Add `pipe-operators` experimental feature" +prs: +- 11131 +--- + +This is a draft implementation of [RFC 0148](https://github.com/NixOS/rfcs/pull/148). + +The `pipe-operators` experimental feature adds [`<|` and `|>` operators][pipe operators] to the Nix language. +*a* `|>` *b* is equivalent to the function application *b* *a*, and +*a* `<|` *b* is equivalent to the function application *a* *b*. + +For example: + +``` +nix-repl> 1 |> builtins.add 2 |> builtins.mul 3 +9 + +nix-repl> builtins.add 1 <| builtins.mul 2 <| 3 +7 +``` + +`<|` and `|>` are right and left associative, respectively, and have lower precedence than any other operator. +These properties may change in future releases. + +See [the RFC](https://github.com/NixOS/rfcs/pull/148) for more examples and rationale. + +[pipe operators]: @docroot@/language/operators.md#pipe-operators diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index 9660a764d..a1e28349b 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -26,13 +26,17 @@ | Logical conjunction (`AND`) | *bool* `&&` *bool* | left | 12 | | Logical disjunction (`OR`) | *bool* \|\| *bool* | left | 13 | | [Logical implication] | *bool* `->` *bool* | right | 14 | +| [Pipe operator] (experimental) | *expr* `\|>` *func* | left | 15 | +| [Pipe operator] (experimental) | *func* `<\|` *expr* | right | 15 | [string]: ./types.md#type-string [path]: ./types.md#type-path -[number]: ./types.md#type-float +[number]: ./types.md#type-float [list]: ./types.md#list [attribute set]: ./types.md#attribute-set + + ## Attribute selection > **Syntax** @@ -182,3 +186,34 @@ Equivalent to `!`*b1* `||` *b2*. [Logical implication]: #logical-implication +## Pipe operators + +- *a* `|>` *b* is equivalent to *b* *a* +- *a* `<|` *b* is equivalent to *a* *b* + +> **Example** +> +> ``` +> nix-repl> 1 |> builtins.add 2 |> builtins.mul 3 +> 9 +> +> nix-repl> builtins.add 1 <| builtins.mul 2 <| 3 +> 7 +> ``` + +> **Warning** +> +> This syntax is part of an +> [experimental feature](@docroot@/contributing/experimental-features.md) +> and may change in future releases. +> +> To use this syntax, make sure the +> [`pipe-operators` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-pipe-operators) +> is enabled. +> For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): +> +> ``` +> extra-experimental-features = pipe-operators +> ``` + +[Pipe operator]: #pipe-operators diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 58401be8e..eb1825b7c 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -67,6 +67,14 @@ static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length) return {result, size_t(t - result)}; } +static void requireExperimentalFeature(const ExperimentalFeature & feature, const Pos & pos) +{ + if (!experimentalFeatureSettings.isEnabled(feature)) + throw ParseError(ErrorInfo{ + .msg = HintFmt("experimental Nix feature '%1%' is disabled; add '--extra-experimental-features %1%' to enable it", showExperimentalFeature(feature)), + .pos = pos, + }); +} } @@ -119,6 +127,12 @@ or { return OR_KW; } \-\> { return IMPL; } \/\/ { return UPDATE; } \+\+ { return CONCAT; } +\<\| { requireExperimentalFeature(Xp::PipeOperators, state->positions[CUR_POS]); + return PIPE_FROM; + } +\|\> { requireExperimentalFeature(Xp::PipeOperators, state->positions[CUR_POS]); + return PIPE_INTO; + } {ID} { yylval->id = {yytext, (size_t) yyleng}; return ID; } {INT} { errno = 0; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 8ea176b24..9ad41c148 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -99,6 +99,14 @@ static void setDocPosition(const LexerState & lexerState, ExprLambda * lambda, P } } +static Expr * makeCall(PosIdx pos, Expr * fn, Expr * arg) { + if (auto e2 = dynamic_cast(fn)) { + e2->args.push_back(arg); + return fn; + } + return new ExprCall(pos, fn, {arg}); +} + %} @@ -123,6 +131,7 @@ static void setDocPosition(const LexerState & lexerState, ExprLambda * lambda, P %type start expr expr_function expr_if expr_op %type expr_select expr_simple expr_app +%type expr_pipe_from expr_pipe_into %type expr_list %type binds %type formals @@ -140,6 +149,7 @@ static void setDocPosition(const LexerState & lexerState, ExprLambda * lambda, P %token PATH HPATH SPATH PATH_END %token URI %token IF THEN ELSE ASSERT WITH LET IN_KW REC INHERIT EQ NEQ AND OR IMPL OR_KW +%token PIPE_FROM PIPE_INTO /* <| and |> */ %token DOLLAR_CURLY /* == ${ */ %token IND_STRING_OPEN IND_STRING_CLOSE %token ELLIPSIS @@ -206,9 +216,21 @@ expr_function expr_if : IF expr THEN expr ELSE expr { $$ = new ExprIf(CUR_POS, $2, $4, $6); } + | expr_pipe_from + | expr_pipe_into | expr_op ; +expr_pipe_from + : expr_op PIPE_FROM expr_pipe_from { $$ = makeCall(state->at(@2), $1, $3); } + | expr_op PIPE_FROM expr_op { $$ = makeCall(state->at(@2), $1, $3); } + ; + +expr_pipe_into + : expr_pipe_into PIPE_INTO expr_op { $$ = makeCall(state->at(@2), $3, $1); } + | expr_op PIPE_INTO expr_op { $$ = makeCall(state->at(@2), $3, $1); } + ; + expr_op : '!' expr_op %prec NOT { $$ = new ExprOpNot($2); } | '-' expr_op %prec NEGATE { $$ = new ExprCall(CUR_POS, new ExprVar(state->s.sub), {new ExprInt(0), $2}); } @@ -233,13 +255,7 @@ expr_op ; expr_app - : expr_app expr_select { - if (auto e2 = dynamic_cast($1)) { - e2->args.push_back($2); - $$ = $1; - } else - $$ = new ExprCall(CUR_POS, $1, {$2}); - } + : expr_app expr_select { $$ = makeCall(CUR_POS, $1, $2); } | expr_select ; diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 1c080e372..a0c955816 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -24,7 +24,7 @@ struct ExperimentalFeatureDetails * feature, we either have no issue at all if few features are not added * at the end of the list, or a proper merge conflict if they are. */ -constexpr size_t numXpFeatures = 1 + static_cast(Xp::VerifiedFetches); +constexpr size_t numXpFeatures = 1 + static_cast(Xp::PipeOperators); constexpr std::array xpFeatureDetails = {{ { @@ -294,6 +294,14 @@ constexpr std::array xpFeatureDetails )", .trackingUrl = "https://github.com/NixOS/nix/milestone/48", }, + { + .tag = Xp::PipeOperators, + .name = "pipe-operators", + .description = R"( + Add `|>` and `<|` operators to the Nix language. + )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/55", + }, }}; static_assert( diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 6ffbc0c10..e65e51280 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -35,6 +35,7 @@ enum struct ExperimentalFeature ConfigurableImpureEnv, MountedSSHStore, VerifiedFetches, + PipeOperators, }; /** diff --git a/tests/functional/lang/eval-fail-pipe-operators.err.exp b/tests/functional/lang/eval-fail-pipe-operators.err.exp new file mode 100644 index 000000000..49f3fa8ad --- /dev/null +++ b/tests/functional/lang/eval-fail-pipe-operators.err.exp @@ -0,0 +1,5 @@ +error: experimental Nix feature 'pipe-operators' is disabled; add '--extra-experimental-features pipe-operators' to enable it + at /pwd/lang/eval-fail-pipe-operators.nix:1:3: + 1| 1 |> 2 + | ^ + 2| diff --git a/tests/functional/lang/eval-fail-pipe-operators.nix b/tests/functional/lang/eval-fail-pipe-operators.nix new file mode 100644 index 000000000..433e0fd7f --- /dev/null +++ b/tests/functional/lang/eval-fail-pipe-operators.nix @@ -0,0 +1 @@ +1 |> 2 diff --git a/tests/unit/libexpr/main.cc b/tests/unit/libexpr/main.cc index cf7fcf5a3..e3412d9ef 100644 --- a/tests/unit/libexpr/main.cc +++ b/tests/unit/libexpr/main.cc @@ -34,6 +34,9 @@ int main (int argc, char **argv) { setEnv("_NIX_TEST_NO_SANDBOX", "1"); #endif + // For pipe operator tests in trivial.cc + experimentalFeatureSettings.set("experimental-features", "pipe-operators"); + ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } diff --git a/tests/unit/libexpr/trivial.cc b/tests/unit/libexpr/trivial.cc index 61ea71a0f..e455a571b 100644 --- a/tests/unit/libexpr/trivial.cc +++ b/tests/unit/libexpr/trivial.cc @@ -182,6 +182,60 @@ namespace nix { ASSERT_THAT(v, IsIntEq(15)); } + TEST_F(TrivialExpressionTest, forwardPipe) { + auto v = eval("1 |> builtins.add 2 |> builtins.mul 3"); + ASSERT_THAT(v, IsIntEq(9)); + } + + TEST_F(TrivialExpressionTest, backwardPipe) { + auto v = eval("builtins.add 1 <| builtins.mul 2 <| 3"); + ASSERT_THAT(v, IsIntEq(7)); + } + + TEST_F(TrivialExpressionTest, forwardPipeEvaluationOrder) { + auto v = eval("1 |> null |> (x: 2)"); + ASSERT_THAT(v, IsIntEq(2)); + } + + TEST_F(TrivialExpressionTest, backwardPipeEvaluationOrder) { + auto v = eval("(x: 1) <| null <| 2"); + ASSERT_THAT(v, IsIntEq(1)); + } + + TEST_F(TrivialExpressionTest, differentPipeOperatorsDoNotAssociate) { + ASSERT_THROW(eval("(x: 1) <| 2 |> (x: 3)"), ParseError); + } + + TEST_F(TrivialExpressionTest, differentPipeOperatorsParensLeft) { + auto v = eval("((x: 1) <| 2) |> (x: 3)"); + ASSERT_THAT(v, IsIntEq(3)); + } + + TEST_F(TrivialExpressionTest, differentPipeOperatorsParensRight) { + auto v = eval("(x: 1) <| (2 |> (x: 3))"); + ASSERT_THAT(v, IsIntEq(1)); + } + + TEST_F(TrivialExpressionTest, forwardPipeLowestPrecedence) { + auto v = eval("false -> true |> (x: !x)"); + ASSERT_THAT(v, IsFalse()); + } + + TEST_F(TrivialExpressionTest, backwardPipeLowestPrecedence) { + auto v = eval("(x: !x) <| false -> true"); + ASSERT_THAT(v, IsFalse()); + } + + TEST_F(TrivialExpressionTest, forwardPipeStrongerThanElse) { + auto v = eval("if true then 1 else 2 |> 3"); + ASSERT_THAT(v, IsIntEq(1)); + } + + TEST_F(TrivialExpressionTest, backwardPipeStrongerThanElse) { + auto v = eval("if true then 1 else 2 <| 3"); + ASSERT_THAT(v, IsIntEq(1)); + } + TEST_F(TrivialExpressionTest, bindOr) { auto v = eval("{ or = 1; }"); ASSERT_THAT(v, IsAttrsOfSize(1)); From 459ee005633eb8094c82b6b6d9bff1df2732b893 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 24 Jul 2024 19:22:53 +0200 Subject: [PATCH 032/142] Render the release notes when building the manual from dev shell --- flake.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/flake.nix b/flake.nix index 2384c2974..fc6d16169 100644 --- a/flake.nix +++ b/flake.nix @@ -333,6 +333,7 @@ ++ [ pkgs.buildPackages.cmake pkgs.buildPackages.shellcheck + pkgs.buildPackages.changelog-d modular.pre-commit.settings.package (pkgs.writeScriptBin "pre-commit-hooks-install" modular.pre-commit.settings.installationScript) From 4bfc96f376ff0e0cd5fba4b36d7afcd4abcee020 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 24 Jul 2024 19:21:34 +0200 Subject: [PATCH 033/142] Fix and update release notes --- doc/manual/rl-next/shebang-relative.md | 2 +- doc/manual/rl-next/zzz-other.md | 50 ++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 doc/manual/rl-next/zzz-other.md diff --git a/doc/manual/rl-next/shebang-relative.md b/doc/manual/rl-next/shebang-relative.md index c887a598a..d12c0f8dc 100644 --- a/doc/manual/rl-next/shebang-relative.md +++ b/doc/manual/rl-next/shebang-relative.md @@ -8,7 +8,7 @@ issues: --- -Relative [path](@docroot@/language/values.md#type-path) literals in `nix-shell` shebang scripts' options are now resolved relative to the [script's location](@docroot@/glossary?highlight=base%20directory#gloss-base-directory). +Relative [path](@docroot@/language/types.md#type-path) literals in `nix-shell` shebang scripts' options are now resolved relative to the [script's location](@docroot@/glossary.md?highlight=base%20directory#gloss-base-directory). Previously they were resolved relative to the current working directory. For example, consider the following script in `~/myproject/say-hi`: diff --git a/doc/manual/rl-next/zzz-other.md b/doc/manual/rl-next/zzz-other.md new file mode 100644 index 000000000..f3721bd38 --- /dev/null +++ b/doc/manual/rl-next/zzz-other.md @@ -0,0 +1,50 @@ +--- +synopsis: Other changes +--- + +- [#9063](https://github.com/NixOS/nix/pull/9063): introduce `libnixflake` and move flakes-specific code there (Nix is internally composed of a set of libraries, and this better reflects the architecture wrt flakes) +- [#10852](https://github.com/NixOS/nix/pull/10852): make Nix commands respond better to interruption - Ctrl+C in the terminal +- [#10853](https://github.com/NixOS/nix/pull/10853): fix docs of `builtins.importNative` +- [#10858](https://github.com/NixOS/nix/pull/10858): `flake check`: Recognize well known `homeModule`/`homeModules` attribute +- [#10865](https://github.com/NixOS/nix/pull/10865): Update dependencies to Nixpkgs 24.05 (when using the `nix` flake), and support bdwgc 8.2.6 ([#11141](https://github.com/NixOS/nix/issues/11141), [#10880](https://github.com/NixOS/nix/pull/10880)) +- [#10883](https://github.com/NixOS/nix/pull/10883): Use `TMP` instead of `XDG_RUNTIME_DIR` +- [#10994](https://github.com/NixOS/nix/pull/10994): fix minor bug in elided item counts when printing values lazily +- [#10988](https://github.com/NixOS/nix/pull/10988): restore `commit-lockfile-summary` alias +- [#10941](https://github.com/NixOS/nix/pull/10941): invalid derivation name now causes an actionable error message +- Changes to the interaction between Nix's I/O coroutines and the garbage collector: +- [#10878](https://github.com/NixOS/nix/pull/10878): allow `ipc-sysv*` in the Darwin build sandbox +- [#11031](https://github.com/NixOS/nix/pull/11031): fix Darwin build sandbox +- [#10907](https://github.com/NixOS/nix/pull/10907): use opaque struct instead of `void *` in the C API +- [#10947](https://github.com/NixOS/nix/issues/10947): fix evaluation cache accidentally persisting disallowed IFD errors +- [#11020](https://github.com/NixOS/nix/pull/11020): enable fetch and eval caching for tarballs +- [#11041](https://github.com/NixOS/nix/pull/11041): add discovered attribute paths to the `--show-trace` error trace in `nix-build`, `nix-env`, OfBorg, and other callers of `getDerivations` +- [#11056](https://github.com/NixOS/nix/pull/11056): `s3` store now uses system defined proxy settings +- [#11077](https://github.com/NixOS/nix/pull/11077): support hardlinks in tarballs +- [#11100](https://github.com/NixOS/nix/pull/11100): pretty print values consistently regardless of prior thunk state +- [#11086](https://github.com/NixOS/nix/pull/11086): fix loss of evaluation cache additions in `nix env run`, `nix shell`, `nix develop`, and `nix fmt` +- [#11149](https://github.com/NixOS/nix/pull/11149): report GC time and number of GC cycles in `NIX_SHOW_STATS=1` report +- [#11142](https://github.com/NixOS/nix/pull/11142): aliased options can now also be passed as flags, just like their "normal" counterparts, e.g. `--build-max-jobs` now works +- [#11043](https://github.com/NixOS/nix/pull/11043): `assert a == b; e` now reports some detail about why `a` and `b` are different when they are +- [#11159](https://github.com/NixOS/nix/pull/11159): don't crash a nix-daemon worker process when the client disconnects +- Stability improvements and fixes [#10861](https://github.com/NixOS/nix/pull/10861), [#10865](https://github.com/NixOS/nix/pull/10865), [#10918](https://github.com/NixOS/nix/pull/10918), [#10916](https://github.com/NixOS/nix/pull/10916), [#10884](https://github.com/NixOS/nix/pull/10884), [#10943](https://github.com/NixOS/nix/pull/10943), [#11019](https://github.com/NixOS/nix/pull/11019), [#11122](https://github.com/NixOS/nix/pull/11122), [#11117](https://github.com/NixOS/nix/pull/11117) +- User documentation improvements [#10888](https://github.com/NixOS/nix/pull/10888), [#10966](https://github.com/NixOS/nix/pull/10966), [#10974](https://github.com/NixOS/nix/pull/10974), [#10997](https://github.com/NixOS/nix/pull/10997), [#11013](https://github.com/NixOS/nix/pull/11013), [#11059](https://github.com/NixOS/nix/pull/11059), [#11119](https://github.com/NixOS/nix/pull/11119), [#11116](https://github.com/NixOS/nix/pull/11116), [#11061](https://github.com/NixOS/nix/pull/11061), [#11102](https://github.com/NixOS/nix/pull/11102) +- BSD support: [#10896](https://github.com/NixOS/nix/pull/10896) [#11022](https://github.com/NixOS/nix/pull/11022) [#11156](https://github.com/NixOS/nix/pull/11156) +- Windows support: [#10769](https://github.com/NixOS/nix/pull/10769), [#10975](https://github.com/NixOS/nix/pull/10975) [#11153](https://github.com/NixOS/nix/pull/11153) +- Portability: [#7048](https://github.com/NixOS/nix/pull/7048) [#11090](https://github.com/NixOS/nix/pull/11090) +- Installer improvements [#10902](https://github.com/NixOS/nix/pull/10902) +- Performance improvements [#10853](https://github.com/NixOS/nix/pull/10853), [#10854](https://github.com/NixOS/nix/pull/10854), [#11082](https://github.com/NixOS/nix/pull/11082), [#11092](https://github.com/NixOS/nix/pull/11092), [#11113](https://github.com/NixOS/nix/pull/11113) + +Contributor experience improvements: + +Use Meson to build Nix (nearing completion) [#10855](https://github.com/NixOS/nix/pull/10855) [#10904](https://github.com/NixOS/nix/pull/10904) [#10908](https://github.com/NixOS/nix/pull/10908) [#10914](https://github.com/NixOS/nix/pull/10914) [#10933](https://github.com/NixOS/nix/pull/10933) [#10936](https://github.com/NixOS/nix/pull/10936) [#10954](https://github.com/NixOS/nix/pull/10954) [#10955](https://github.com/NixOS/nix/pull/10955) [#10967](https://github.com/NixOS/nix/pull/10967) [#10963](https://github.com/NixOS/nix/pull/10963) [#10973](https://github.com/NixOS/nix/pull/10973) [#11034](https://github.com/NixOS/nix/pull/11034) [#11054](https://github.com/NixOS/nix/pull/11054) [#11055](https://github.com/NixOS/nix/pull/11055) [#11064](https://github.com/NixOS/nix/pull/11064) [#11060](https://github.com/NixOS/nix/pull/11060) [#11155](https://github.com/NixOS/nix/pull/11155) +- Testing improvements [#10864](https://github.com/NixOS/nix/pull/10864), [#10903](https://github.com/NixOS/nix/pull/10903), [#10874](https://github.com/NixOS/nix/pull/10874), [#10922](https://github.com/NixOS/nix/pull/10922), [#11006](https://github.com/NixOS/nix/pull/11006), [#11110](https://github.com/NixOS/nix/pull/11110), [#10931](https://github.com/NixOS/nix/pull/10931), [#11123](https://github.com/NixOS/nix/pull/11123) + - [#10603](https://github.com/NixOS/nix/pull/10603): We now evaluate a set of flakes in CI + - [#10922](https://github.com/NixOS/nix/pull/10922): The functional test suite is now run in both in the build sandbox and in a NixOS environment +- CI improvements [#10929](https://github.com/NixOS/nix/pull/10929) [#10999](https://github.com/NixOS/nix/pull/10999) [#11009](https://github.com/NixOS/nix/pull/11009) [#11065](https://github.com/NixOS/nix/pull/11065) [#11071](https://github.com/NixOS/nix/pull/11071) +- Contributor documentation improvements [#10869](https://github.com/NixOS/nix/pull/10869), [#9871](https://github.com/NixOS/nix/pull/9871), [#10960](https://github.com/NixOS/nix/pull/10960), [#11147](https://github.com/NixOS/nix/pull/11147) +- Error message improvements: [#11050](https://github.com/NixOS/nix/pull/11050) [#11154](https://github.com/NixOS/nix/pull/11154) +- Cleaning up the Settings system (`nix.conf` and related architectural cleanups): [#10913](https://github.com/NixOS/nix/pull/10913), [#10951](https://github.com/NixOS/nix/pull/10951), [#11007](https://github.com/NixOS/nix/pull/11007), [#11108](https://github.com/NixOS/nix/pull/11108), [#11014](https://github.com/NixOS/nix/pull/11014), [#11109](https://github.com/NixOS/nix/pull/11109), [#11112](https://github.com/NixOS/nix/pull/11112) +- Other cleanups and refactors [#10857](https://github.com/NixOS/nix/pull/10857) [#10935](https://github.com/NixOS/nix/pull/10935) [#10873](https://github.com/NixOS/nix/pull/10873) [#10745](https://github.com/NixOS/nix/pull/10745) [#10961](https://github.com/NixOS/nix/pull/10961) [#10962](https://github.com/NixOS/nix/pull/10962) [#10972](https://github.com/NixOS/nix/pull/10972) [#11018](https://github.com/NixOS/nix/pull/11018) [#11035](https://github.com/NixOS/nix/pull/11035) [#11037](https://github.com/NixOS/nix/pull/11037) [#11081](https://github.com/NixOS/nix/pull/11081) [#11089](https://github.com/NixOS/nix/pull/11089) [#11093](https://github.com/NixOS/nix/pull/11093) [#11114](https://github.com/NixOS/nix/pull/11114) [#11103](https://github.com/NixOS/nix/pull/11103) [#11126](https://github.com/NixOS/nix/pull/11126) [#11125](https://github.com/NixOS/nix/pull/11125) [#11120](https://github.com/NixOS/nix/pull/11120) +- Scheduler/builder refactoring [#11005](https://github.com/NixOS/nix/pull/11005) +- [#11011](https://github.com/NixOS/nix/pull/11011): enable `-Werror=unused-result` + From caf4e98f0c0a0a7178538ee8ef7b199d2a655aac Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Jul 2024 20:10:41 +0200 Subject: [PATCH 034/142] Log download durations --- src/libstore/filetransfer.cc | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index cbbb0fe7a..e7dae333b 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -71,7 +71,10 @@ struct curlFileTransfer : public FileTransfer curl_off_t writtenToSink = 0; + std::chrono::steady_clock::time_point startTime = std::chrono::steady_clock::now(); + inline static const std::set successfulStatuses {200, 201, 204, 206, 304, 0 /* other protocol */}; + /* Get the HTTP status code, or 0 for other protocols. */ long getHTTPStatus() { @@ -373,10 +376,14 @@ struct curlFileTransfer : public FileTransfer void finish(CURLcode code) { + auto finishTime = std::chrono::steady_clock::now(); + auto httpStatus = getHTTPStatus(); - debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes", - request.verb(), request.uri, code, httpStatus, result.bodySize); + debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes, duration = %.2f s", + request.verb(), request.uri, code, httpStatus, result.bodySize, + std::chrono::duration_cast(finishTime - startTime).count() / 1000.0f + ); appendCurrentUrl(); From 8ffea0a018874e60584eabeb620ec3495873c30d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Jul 2024 20:10:45 +0200 Subject: [PATCH 035/142] Add 'download-buffer-size' setting We are piping curl downloads into `unpackTarfileToSink()`, but the latter is typically slower than the former if you're on a fast connection. So the download could appear unnecessarily slow. (There is even a risk that if the Git import is *really* slow for whatever reason, the TCP connection could time out.) So let's make the download buffer bigger by default - 64 MiB is big enough for the Nixpkgs tarball. Perhaps in the future, we could have an unlimited buffer that spills data to disk beyond a certain threshold, but that's probably overkill. --- src/libstore/filetransfer.cc | 2 +- src/libstore/filetransfer.hh | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index e7dae333b..f48c2e22d 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -858,7 +858,7 @@ void FileTransfer::download( buffer). We don't wait forever to prevent stalling the download thread. (Hopefully sleeping will throttle the sender.) */ - if (state->data.size() > 1024 * 1024) { + if (state->data.size() > fileTransferSettings.downloadBufferSize) { debug("download buffer is full; going to sleep"); state.wait_for(state->request, std::chrono::seconds(10)); } diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index 1f5b4ab93..d836ab2c4 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -47,6 +47,12 @@ struct FileTransferSettings : Config Setting tries{this, 5, "download-attempts", "How often Nix will attempt to download a file before giving up."}; + + Setting downloadBufferSize{this, 64 * 1024 * 1024, "download-buffer-size", + R"( + The size of Nix's internal download buffer during `curl` transfers. If data is + not processed quickly enough to exceed the size of this buffer, downloads may stall. + )"}; }; extern FileTransferSettings fileTransferSettings; From f6a9a71b38b25d6c9fb0b9a7fbf0eccf99fa5520 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Jul 2024 20:14:31 +0200 Subject: [PATCH 036/142] Warn if the download buffer is full --- src/libstore/filetransfer.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index f48c2e22d..5ea8b6f96 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -860,6 +860,8 @@ void FileTransfer::download( sender.) */ if (state->data.size() > fileTransferSettings.downloadBufferSize) { debug("download buffer is full; going to sleep"); + static bool haveWarned = false; + warnOnce(haveWarned, "download buffer is full; consider increasing the 'download-buffer-size' setting"); state.wait_for(state->request, std::chrono::seconds(10)); } From 01839b525c5e80d7f67f2808f7f7fb478ddf1ba0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Jul 2024 20:22:26 +0200 Subject: [PATCH 037/142] Show when we're unpacking an archive into the Git cache This happens in parallel with the download (which starts later), so you only see this message when the download has finished but the import hasn't. --- src/libfetchers/github.cc | 5 +++++ src/libfetchers/tarball.cc | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 2968d2df2..5710b94d5 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -254,10 +254,15 @@ struct GitArchiveInputScheme : InputScheme getFileTransfer()->download(std::move(req), sink); }); + auto act = std::make_unique(*logger, lvlInfo, actUnknown, + fmt("unpacking '%s' into the Git cache", input.to_string())); + TarArchive archive { *source }; auto parseSink = getTarballCache()->getFileSystemObjectSink(); auto lastModified = unpackTarfileToSink(archive, *parseSink); + act.reset(); + TarballInfo tarballInfo { .treeHash = parseSink->sync(), .lastModified = lastModified diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 55db3eafb..5837cf8a4 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -143,6 +143,9 @@ DownloadTarballResult downloadTarball( // TODO: fall back to cached value if download fails. + auto act = std::make_unique(*logger, lvlInfo, actUnknown, + fmt("unpacking '%s' into the Git cache", url)); + AutoDelete cleanupTemp; /* Note: if the download is cached, `importTarball()` will receive @@ -167,6 +170,8 @@ DownloadTarballResult downloadTarball( auto parseSink = getTarballCache()->getFileSystemObjectSink(); auto lastModified = unpackTarfileToSink(archive, *parseSink); + act.reset(); + auto res(_res->lock()); Attrs infoAttrs; From e0620213146b1a581533302a2fab54af21d49685 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 24 Jul 2024 23:17:15 +0200 Subject: [PATCH 038/142] fix `NIX_PATH` for real (#11079) * fix NIX_PATH overriding - test restricted evaluation - test precedence for setting the search path Co-authored-by: Robert Hensing Co-authored-by: John Ericson --- src/libexpr/eval-gc.cc | 8 ++++ src/libexpr/eval-settings.cc | 7 +--- src/libexpr/eval-settings.hh | 4 +- src/libexpr/eval.cc | 17 +++++++-- src/libutil/config.cc | 14 ++++++- tests/functional/nix_path.sh | 70 ++++++++++++++++++++++++++++++++++ tests/functional/restricted.sh | 3 ++ 7 files changed, 111 insertions(+), 12 deletions(-) diff --git a/src/libexpr/eval-gc.cc b/src/libexpr/eval-gc.cc index ba19cd74e..d82ed1534 100644 --- a/src/libexpr/eval-gc.cc +++ b/src/libexpr/eval-gc.cc @@ -1,5 +1,7 @@ #include "error.hh" #include "environment-variables.hh" +#include "eval-settings.hh" +#include "config-global.hh" #include "serialise.hh" #include "eval-gc.hh" @@ -230,6 +232,12 @@ void initGC() gcCyclesAfterInit = GC_get_gc_no(); #endif + // NIX_PATH must override the regular setting + // See the comment in applyConfig + if (auto nixPathEnv = getEnv("NIX_PATH")) { + globalConfig.set("nix-path", concatStringsSep(" ", EvalSettings::parseNixPath(nixPathEnv.value()))); + } + gcInitialised = true; } diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc index eb5761638..2846eccbc 100644 --- a/src/libexpr/eval-settings.cc +++ b/src/libexpr/eval-settings.cc @@ -8,7 +8,7 @@ namespace nix { /* Very hacky way to parse $NIX_PATH, which is colon-separated, but can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */ -static Strings parseNixPath(const std::string & s) +Strings EvalSettings::parseNixPath(const std::string & s) { Strings res; @@ -48,10 +48,7 @@ EvalSettings::EvalSettings(bool & readOnlyMode, EvalSettings::LookupPathHooks lo : readOnlyMode{readOnlyMode} , lookupPathHooks{lookupPathHooks} { - auto var = getEnv("NIX_PATH"); - if (var) nixPath = parseNixPath(*var); - - var = getEnv("NIX_ABORT_ON_WARN"); + auto var = getEnv("NIX_ABORT_ON_WARN"); if (var && (var == "1" || var == "yes" || var == "true")) builtinsAbortOnWarn = true; } diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index 89a42caba..8f48b53a5 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -47,6 +47,8 @@ struct EvalSettings : Config static bool isPseudoUrl(std::string_view s); + static Strings parseNixPath(const std::string & s); + static std::string resolvePseudoUrl(std::string_view url); LookupPathHooks lookupPathHooks; @@ -71,7 +73,7 @@ struct EvalSettings : Config )"}; Setting nixPath{ - this, getDefaultNixPath(), "nix-path", + this, {}, "nix-path", R"( List of search paths to use for [lookup path](@docroot@/language/constructs/lookup-path.md) resolution. This setting determines the value of diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index b192f9b4b..9a6b6c9e8 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -215,7 +215,7 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env) static constexpr size_t BASE_ENV_SIZE = 128; EvalState::EvalState( - const LookupPath & _lookupPath, + const LookupPath & lookupPathFromArguments, ref store, const fetchers::Settings & fetchSettings, const EvalSettings & settings, @@ -331,12 +331,21 @@ EvalState::EvalState( vStringSymlink.mkString("symlink"); vStringUnknown.mkString("unknown"); - /* Initialise the Nix expression search path. */ + /* Construct the Nix expression search path. */ + assert(lookupPath.elements.empty()); if (!settings.pureEval) { - for (auto & i : _lookupPath.elements) + for (auto & i : lookupPathFromArguments.elements) { lookupPath.elements.emplace_back(LookupPath::Elem {i}); - for (auto & i : settings.nixPath.get()) + } + /* $NIX_PATH overriding regular settings is implemented as a hack in `initGC()` */ + for (auto & i : settings.nixPath.get()) { lookupPath.elements.emplace_back(LookupPath::Elem::parse(i)); + } + if (!settings.restrictEval) { + for (auto & i : EvalSettings::getDefaultNixPath()) { + lookupPath.elements.emplace_back(LookupPath::Elem::parse(i)); + } + } } /* Allow access to all paths in the search path. */ diff --git a/src/libutil/config.cc b/src/libutil/config.cc index b39948261..ca8480304 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -1,6 +1,7 @@ #include "config.hh" #include "args.hh" #include "abstract-setting-to-json.hh" +#include "environment-variables.hh" #include "experimental-features.hh" #include "util.hh" #include "file-system.hh" @@ -170,9 +171,18 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string set(name, value); // Then apply other settings - for (const auto & [name, value] : parsedContents) - if (name != "experimental-features" && name != "extra-experimental-features") + // XXX: NIX_PATH must override the regular setting! This is done in `initGC()` + // Environment variables overriding settings should probably be part of the Config mechanism, + // but at the time of writing it's not worth building that for just one thing + for (const auto & [name, value] : parsedContents) { + if (name != "experimental-features" && name != "extra-experimental-features") { + if ((name == "nix-path" || name == "extra-nix-path") + && getEnv("NIX_PATH").has_value()) { + continue; + } set(name, value); + } + } } void Config::resetOverridden() diff --git a/tests/functional/nix_path.sh b/tests/functional/nix_path.sh index e6a2193f3..7e6a0458d 100755 --- a/tests/functional/nix_path.sh +++ b/tests/functional/nix_path.sh @@ -14,3 +14,73 @@ nix-instantiate --eval -E '' --restrict-eval [[ $(nix-instantiate --find-file by-absolute-path/simple.nix) = $PWD/simple.nix ]] [[ $(nix-instantiate --find-file by-relative-path/simple.nix) = $PWD/simple.nix ]] + +# this is the human-readable specification for the following test cases of interactions between various ways of specifying NIX_PATH. +# TODO: the actual tests are incomplete and too manual. +# there should be 43 of them, since the table has 9 rows and columns, and 2 interactions are meaningless +# ideally they would work off the table programmatically. +# +# | precedence | hard-coded | nix-path in file | extra-nix-path in file | nix-path in env | extra-nix-path in env | NIX_PATH | nix-path | extra-nix-path | -I | +# |------------------------|------------|------------------|------------------------|-----------------|-----------------------|-----------|-----------|-----------------|-----------------| +# | hard-coded | x | ^override | ^append | ^override | ^append | ^override | ^override | ^append | ^append | +# | nix-path in file | | last wins | ^append | ^override | ^append | ^override | ^override | ^append | ^append | +# | extra-nix-path in file | | | append in order | ^override | ^append | ^override | ^override | ^append | ^append | +# | nix-path in env | | | | last wins | ^append | ^override | ^override | ^append | ^append | +# | extra-nix-path in env | | | | | append in order | ^override | ^override | ^append | ^append | +# | NIX_PATH | | | | | | x | ^override | ^append | ^append | +# | nix-path | | | | | | | last wins | ^append | ^append | +# | extra-nix-path | | | | | | | | append in order | append in order | +# | -I | | | | | | | | | append in order | + +unset NIX_PATH + +mkdir -p $TEST_ROOT/{from-nix-path-file,from-NIX_PATH,from-nix-path,from-extra-nix-path,from-I} +for i in from-nix-path-file from-NIX_PATH from-nix-path from-extra-nix-path from-I; do + touch $TEST_ROOT/$i/only-$i.nix +done + +# finding something that's not in any of the default paths fails +( ! $(nix-instantiate --find-file test) ) + +echo "nix-path = test=$TEST_ROOT/from-nix-path-file" >> "$test_nix_conf" + +# Use nix.conf in absence of NIX_PATH +[[ $(nix-instantiate --find-file test) = $TEST_ROOT/from-nix-path-file ]] + +# NIX_PATH overrides nix.conf +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --find-file test) = $TEST_ROOT/from-NIX_PATH ]] +# if NIX_PATH does not have the desired entry, it fails +(! NIX_PATH=test=$TEST_ROOT nix-instantiate --find-file test/only-from-nix-path-file.nix) + +# -I extends nix.conf +[[ $(nix-instantiate -I test=$TEST_ROOT/from-I --find-file test/only-from-I.nix) = $TEST_ROOT/from-I/only-from-I.nix ]] +# if -I does not have the desired entry, the value from nix.conf is used +[[ $(nix-instantiate -I test=$TEST_ROOT/from-I --find-file test/only-from-nix-path-file.nix) = $TEST_ROOT/from-nix-path-file/only-from-nix-path-file.nix ]] + +# -I extends NIX_PATH +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate -I test=$TEST_ROOT/from-I --find-file test/only-from-I.nix) = $TEST_ROOT/from-I/only-from-I.nix ]] +# if -I does not have the desired entry, the value from NIX_PATH is used +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate -I test=$TEST_ROOT/from-I --find-file test/only-from-NIX_PATH.nix) = $TEST_ROOT/from-NIX_PATH/only-from-NIX_PATH.nix ]] + +# --extra-nix-path extends NIX_PATH +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --extra-nix-path test=$TEST_ROOT/from-extra-nix-path --find-file test/only-from-extra-nix-path.nix) = $TEST_ROOT/from-extra-nix-path/only-from-extra-nix-path.nix ]] +# if --extra-nix-path does not have the desired entry, the value from NIX_PATH is used +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --extra-nix-path test=$TEST_ROOT/from-extra-nix-path --find-file test/only-from-NIX_PATH.nix) = $TEST_ROOT/from-NIX_PATH/only-from-NIX_PATH.nix ]] + +# --nix-path overrides NIX_PATH +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path --find-file test) = $TEST_ROOT/from-nix-path ]] +# if --nix-path does not have the desired entry, it fails +(! NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path --find-file test/only-from-NIX_PATH.nix) + +# --nix-path overrides nix.conf +[[ $(nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path --find-file test) = $TEST_ROOT/from-nix-path ]] +(! nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path --find-file test/only-from-nix-path-file.nix) + +# --extra-nix-path extends nix.conf +[[ $(nix-instantiate --extra-nix-path test=$TEST_ROOT/from-extra-nix-path --find-file test/only-from-extra-nix-path.nix) = $TEST_ROOT/from-extra-nix-path/only-from-extra-nix-path.nix ]] +# if --extra-nix-path does not have the desired entry, it is taken from nix.conf +[[ $(nix-instantiate --extra-nix-path test=$TEST_ROOT/from-extra-nix-path --find-file test) = $TEST_ROOT/from-nix-path-file ]] + +# -I extends --nix-path +[[ $(nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path -I test=$TEST_ROOT/from-I --find-file test/only-from-I.nix) = $TEST_ROOT/from-I/only-from-I.nix ]] +[[ $(nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path -I test=$TEST_ROOT/from-I --find-file test/only-from-nix-path.nix) = $TEST_ROOT/from-nix-path/only-from-nix-path.nix ]] diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh index 915d973b0..591367e9f 100755 --- a/tests/functional/restricted.sh +++ b/tests/functional/restricted.sh @@ -10,6 +10,9 @@ nix-instantiate --restrict-eval --eval -E '1 + 2' nix-instantiate --restrict-eval ./simple.nix -I src=. nix-instantiate --restrict-eval ./simple.nix -I src1=simple.nix -I src2=config.nix -I src3=./simple.builder.sh +# no default NIX_PATH +(unset NIX_PATH; ! nix-instantiate --restrict-eval --find-file .) + (! nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix') nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I src=../.. From dba1142c01111f1f98cc89aa850d5df0c012a907 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 25 Jul 2024 03:45:34 +0200 Subject: [PATCH 039/142] docs: add identifiers (#11174) * docs: add identifiers * clarify attribute set notation and add examples * add definition of names Co-authored-by: Ryan Hendrickson --- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/language/identifiers.md | 50 ++++++++++++++++ doc/manual/src/language/operators.md | 6 -- doc/manual/src/language/syntax.md | 83 +++++++++++++++++++------- 4 files changed, 112 insertions(+), 28 deletions(-) create mode 100644 doc/manual/src/language/identifiers.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index a6a2101e9..b4dd277e3 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -28,6 +28,7 @@ - [Data Types](language/types.md) - [String context](language/string-context.md) - [Syntax and semantics](language/syntax.md) + - [Identifiers](language/identifiers.md) - [Scoping rules](language/scope.md) - [String interpolation](language/string-interpolation.md) - [Lookup path](language/constructs/lookup-path.md) diff --git a/doc/manual/src/language/identifiers.md b/doc/manual/src/language/identifiers.md new file mode 100644 index 000000000..c9e981da6 --- /dev/null +++ b/doc/manual/src/language/identifiers.md @@ -0,0 +1,50 @@ +# Identifiers + +An *identifier* is an [ASCII](https://en.wikipedia.org/wiki/ASCII) character sequence that: +- Starts with a letter (`a-z`, `A-Z`) or underscore (`_`) +- Can contain any number of: + - Letters (`a-z`, `A-Z`) + - Digits (`0-9`) + - Underscores (`_`) + - Apostrophes (`'`) + - Hyphens (`-`) +- Is not one of the [keywords](#keywords) + +> **Syntax** +> +> *identifier* ~ `[A-Za-z_][A-Za-z0-9_'-]*` + +# Names + +A name can be an [identifier](#identifier) or a [string literal](./syntax.md#string-literal). + +> **Syntax** +> +> *name* → *identifier* | *string* + +Names are used in [attribute sets](./syntax.md#attrs-literal), [`let` bindings](./syntax.md#let-expressions), and [`inherit`](./syntax.md#inheriting attributes). + +# Keywords + +These keywords are reserved and cannot be used as [identifiers](#identifiers): + +- [`assert`](./syntax.md#assertions) +- [`else`][if] +- [`if`][if] +- [`in`][let] +- [`inherit`](./syntax.md#inheriting-attributes) +- [`let`][let] +- [`or`](./operators.md#attribute-selection) (see note) +- [`rec`](./syntax.md#recursive-sets) +- [`then`][if] +- [`with`](./syntax.md#with-expressions) + +[if]: ./syntax.md#conditionals +[let]: ./syntax.md#let-expressions + +> **Note** +> +> The Nix language evaluator currently allows `or` to be used as a name in some contexts, for backwards compatibility reasons. +> Users are advised not to rely on this. +> +> There are long-standing issues with how `or` is parsed as a name, which can't be resolved without making a breaking change to the language. diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index 9660a764d..d2476c413 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -42,12 +42,6 @@ Select the attribute denoted by attribute path *attrpath* from [attribute set] *attrset*. If the attribute doesn’t exist, return the *expr* after `or` if provided, otherwise abort evaluation. -An attribute path is a dot-separated list of [attribute names](./types.md#attribute-set). - -> **Syntax** -> -> *attrpath* = *name* [ `.` *name* ]... - [Attribute selection]: #attribute-selection ## Has attribute diff --git a/doc/manual/src/language/syntax.md b/doc/manual/src/language/syntax.md index b0779ea95..6108bacd6 100644 --- a/doc/manual/src/language/syntax.md +++ b/doc/manual/src/language/syntax.md @@ -247,37 +247,76 @@ Elements in a list can be accessed using [`builtins.elemAt`](./builtins.md#built ## Attribute Set {#attrs-literal} -An attribute set is a collection of name-value-pairs (called *attributes*) enclosed in curly brackets (`{ }`). +An attribute set is a collection of name-value-pairs called *attributes*. -An attribute name can be an identifier or a [string](#string). -An identifier must start with a letter (`a-z`, `A-Z`) or underscore (`_`), and can otherwise contain letters (`a-z`, `A-Z`), numbers (`0-9`), underscores (`_`), apostrophes (`'`), or dashes (`-`). +Attribute sets are written enclosed in curly brackets (`{ }`). +Attribute names and attribute values are separated by an equal sign (`=`). +Each value can be an arbitrary expression, terminated by a semicolon (`;`) + +An attribute name is a string without context, and is denoted by a [name] (an [identifier](./identifiers.md#identifiers) or [string literal](#string-literal)). + +[name]: ./identifiers.md#names > **Syntax** > -> *name* = *identifier* | *string* \ -> *identifier* ~ `[a-zA-Z_][a-zA-Z0-9_'-]*` - -Names and values are separated by an equal sign (`=`). -Each value is an arbitrary expression terminated by a semicolon (`;`). - -> **Syntax** -> -> *attrset* = `{` [ *name* `=` *expr* `;` ]... `}` +> *attrset* → `{` { *name* `=` *expr* `;` } `}` Attributes can appear in any order. -An attribute name may only occur once. +An attribute name may only occur once in each attribute set. -Example: +> **Example** +> +> This defines an attribute set with attributes named: +> - `x` with the value `123`, an integer +> - `text` with the value `"Hello"`, a string +> - `y` where the value is the result of applying the function `f` to the attribute set `{ bla = 456; }` +> +> ```nix +> { +> x = 123; +> text = "Hello"; +> y = f { bla = 456; }; +> } +> ``` -```nix -{ - x = 123; - text = "Hello"; - y = f { bla = 456; }; -} -``` +Attributes in nested attribute sets can be written using *attribute paths*. -This defines a set with attributes named `x`, `text`, `y`. +> **Syntax** +> +> *attrset* → `{` { *attrpath* `=` *expr* `;` } `}` + +An attribute path is a dot-separated list of [names][name]. + +> **Syntax** +> +> *attrpath* = *name* { `.` *name* } + + + +> **Example** +> +> ```nix +> { a.b.c = 1; a.b.d = 2; } +> ``` +> +> { +> a = { +> b = { +> c = 1; +> d = 2; +> }; +> }; +> } + +Attribute names can also be set implicitly by using the [`inherit` keyword](#inheriting-attributes). + +> **Example** +> +> ```nix +> { inherit (builtins) true; } +> ``` +> +> { true = true; } Attributes can be accessed with the [`.` operator](./operators.md#attribute-selection). From f4915af71c9a72a0519c90088abb3a36c265ca95 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 24 Jul 2024 22:25:08 -0400 Subject: [PATCH 040/142] Put flake functional tests in their own group This is a nice thing to have, and it made it easier to work on the Meson-ifcation of these functional tests too. --- Makefile | 1 + tests/functional/flakes/local.mk | 24 ++++++++++++++++++++++++ tests/functional/local.mk | 21 --------------------- 3 files changed, 25 insertions(+), 21 deletions(-) create mode 100644 tests/functional/flakes/local.mk diff --git a/Makefile b/Makefile index bb64a104e..d3ea8513d 100644 --- a/Makefile +++ b/Makefile @@ -54,6 +54,7 @@ ifeq ($(ENABLE_FUNCTIONAL_TESTS), yes) ifdef HOST_UNIX makefiles += \ tests/functional/local.mk \ + tests/functional/flakes/local.mk \ tests/functional/ca/local.mk \ tests/functional/git-hashing/local.mk \ tests/functional/dyn-drv/local.mk \ diff --git a/tests/functional/flakes/local.mk b/tests/functional/flakes/local.mk new file mode 100644 index 000000000..71e50ad07 --- /dev/null +++ b/tests/functional/flakes/local.mk @@ -0,0 +1,24 @@ +flake-tests := \ + $(d)/flakes.sh \ + $(d)/develop.sh \ + $(d)/edit.sh \ + $(d)/run.sh \ + $(d)/mercurial.sh \ + $(d)/circular.sh \ + $(d)/init.sh \ + $(d)/inputs.sh \ + $(d)/follow-paths.sh \ + $(d)/bundle.sh \ + $(d)/check.sh \ + $(d)/unlocked-override.sh \ + $(d)/absolute-paths.sh \ + $(d)/absolute-attr-paths.sh \ + $(d)/build-paths.sh \ + $(d)/flake-in-submodule.sh \ + $(d)/prefetch.sh \ + $(d)/eval-cache.sh \ + $(d)/search-root.sh \ + $(d)/config.sh \ + $(d)/show.sh + +install-tests-groups += flake diff --git a/tests/functional/local.mk b/tests/functional/local.mk index 797002e92..8b4945cac 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -1,23 +1,5 @@ nix_tests = \ test-infra.sh \ - flakes/flakes.sh \ - flakes/develop.sh \ - flakes/edit.sh \ - flakes/run.sh \ - flakes/mercurial.sh \ - flakes/circular.sh \ - flakes/init.sh \ - flakes/inputs.sh \ - flakes/follow-paths.sh \ - flakes/bundle.sh \ - flakes/check.sh \ - flakes/unlocked-override.sh \ - flakes/absolute-paths.sh \ - flakes/absolute-attr-paths.sh \ - flakes/build-paths.sh \ - flakes/flake-in-submodule.sh \ - flakes/prefetch.sh \ - flakes/eval-cache.sh \ gc.sh \ nix-collect-garbage-d.sh \ remote-store.sh \ @@ -61,7 +43,6 @@ nix_tests = \ restricted.sh \ fetchGitSubmodules.sh \ fetchGitVerification.sh \ - flakes/search-root.sh \ readfile-context.sh \ nix-channel.sh \ recursive.sh \ @@ -102,7 +83,6 @@ nix_tests = \ nix-copy-ssh-ng.sh \ post-hook.sh \ function-trace.sh \ - flakes/config.sh \ fmt.sh \ eval-store.sh \ why-depends.sh \ @@ -125,7 +105,6 @@ nix_tests = \ store-info.sh \ fetchClosure.sh \ completions.sh \ - flakes/show.sh \ impure-derivations.sh \ path-from-hash-part.sh \ path-info.sh \ From dcbe2453f536464a4747c20a057df3ea93c0c400 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 24 Jul 2024 22:36:43 -0400 Subject: [PATCH 041/142] Change skipped test error code from 99 to 77 Meson uses a venerable GNU convention described in https://www.gnu.org/software/automake/manual/html_node/Scripts_002dbased-Testsuites.html in which: > When no test protocol is in use, an exit status of 0 from a test > script will denote a success, an exit status of 77 a skipped test, an > exit status of 99 a hard error, and any other exit status will denote > a failure. 77 is thus what we want, not 99. --- mk/run-test.sh | 2 +- tests/functional/common/vars-and-functions.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mk/run-test.sh b/mk/run-test.sh index 543c845e1..7f9f1d5f8 100755 --- a/mk/run-test.sh +++ b/mk/run-test.sh @@ -28,7 +28,7 @@ run_test if [[ "$status" = 0 ]]; then echo "$post_run_msg [${green}PASS$normal]" -elif [[ "$status" = 99 ]]; then +elif [[ "$status" = 77 ]]; then echo "$post_run_msg [${yellow}SKIP$normal]" else echo "$post_run_msg [${red}FAIL$normal]" diff --git a/tests/functional/common/vars-and-functions.sh b/tests/functional/common/vars-and-functions.sh index 4316a30d5..6a0988f12 100644 --- a/tests/functional/common/vars-and-functions.sh +++ b/tests/functional/common/vars-and-functions.sh @@ -190,7 +190,7 @@ isDaemonNewer () { skipTest () { echo "$1, skipping this test..." >&2 - exit 99 + exit 77 } TODO_NixOS() { From a2fed6db9e6ec0ce8441d16bec0752b420dd49e9 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 25 Jul 2024 04:53:06 +0200 Subject: [PATCH 042/142] manual: Contributing -> Development, Hacking -> Building (#9014) * manual: Contributing -> Development, Hacking -> Building what's currently called "hacking" are really instructions for setting up a development environment and compiling from source. we have a contribution guide in the repo (which rightly focuses on GitHub workflows), and the material in the manual is more about working on the code itself. since we'd otherwise have three headings that amount to "Building Nix", this change also moves the "classic Nix" instructions to the top. we may want to reorganise this in the future, and bring contributor-oriented information closer to the code, but for now let's stick to more accurate names to ease navigation. --- CONTRIBUTING.md | 8 +- README.md | 7 +- doc/manual/generate-builtins.nix | 2 +- doc/manual/generate-manpage.nix | 2 +- doc/manual/generate-settings.nix | 4 +- doc/manual/generate-store-info.nix | 4 +- doc/manual/generate-xp-features-shortlist.nix | 2 +- doc/manual/local.mk | 8 +- doc/manual/redirects.js | 11 +- doc/manual/src/SUMMARY.md.in | 17 +- doc/manual/src/_redirects | 10 +- doc/manual/src/c-api.md | 2 +- .../src/command-ref/experimental-commands.md | 2 +- .../src/command-ref/nix-store/realise.md | 2 +- .../hacking.md => development/building.md} | 181 +++++------------- .../cli-guideline.md | 0 doc/manual/src/development/contributing.md | 79 ++++++++ .../src/{contributing => development}/cxx.md | 0 .../documentation.md | 2 +- .../experimental-features.md | 0 .../{contributing => development}/index.md | 2 +- .../json-guideline.md | 0 .../{contributing => development}/testing.md | 0 doc/manual/src/glossary.md | 4 +- .../src/installation/installing-binary.md | 2 +- .../src/language/advanced-attributes.md | 10 +- doc/manual/src/protocols/derivation-aterm.md | 4 +- doc/manual/src/protocols/json/derivation.md | 2 +- .../src/protocols/json/store-object-info.md | 2 +- doc/manual/src/release-notes/rl-2.18.md | 4 +- doc/manual/src/release-notes/rl-2.19.md | 6 +- doc/manual/src/release-notes/rl-2.23.md | 4 +- doc/manual/src/release-notes/rl-2.4.md | 2 +- .../file-system-object/content-address.md | 2 +- .../src/store/store-object/content-address.md | 2 +- src/libexpr/eval-settings.hh | 2 +- src/libexpr/primops/fetchTree.cc | 10 +- src/libstore/globals.hh | 6 +- src/libutil/config.hh | 2 +- src/nix/nix.md | 6 +- 40 files changed, 214 insertions(+), 201 deletions(-) rename doc/manual/src/{contributing/hacking.md => development/building.md} (74%) rename doc/manual/src/{contributing => development}/cli-guideline.md (100%) create mode 100644 doc/manual/src/development/contributing.md rename doc/manual/src/{contributing => development}/cxx.md (100%) rename doc/manual/src/{contributing => development}/documentation.md (99%) rename doc/manual/src/{contributing => development}/experimental-features.md (100%) rename doc/manual/src/{contributing => development}/index.md (77%) rename doc/manual/src/{contributing => development}/json-guideline.md (100%) rename doc/manual/src/{contributing => development}/testing.md (100%) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 38f5d43b7..12423366a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,9 +41,9 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). There are many open pull requests that might already do what you intend to work on. You can use [labels](https://github.com/NixOS/nix/labels) to filter for relevant topics. -3. Check the [Nix reference manual](https://nixos.org/manual/nix/unstable/contributing/hacking.html) for information on building Nix and running its tests. +3. Check the [Nix reference manual](https://nix.dev/manual/nix/development/development/building.html) for information on building Nix and running its tests. - For contributions to the command line interface, please check the [CLI guidelines](https://nixos.org/manual/nix/unstable/contributing/cli-guideline.html). + For contributions to the command line interface, please check the [CLI guidelines](https://nix.dev/manual/nix/development/development/cli-guideline.html). 4. Make your change! @@ -69,7 +69,7 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). - [ ] API documentation in header files - [ ] Code and comments are self-explanatory - [ ] Commit message explains **why** the change was made - - [ ] New feature or incompatible change: [add a release note](https://nixos.org/manual/nix/stable/contributing/hacking#add-a-release-note) + - [ ] New feature or incompatible change: [add a release note](https://nix.dev/manual/nix/development/development/contributing.html#add-a-release-note) 7. If you need additional feedback or help to getting pull request into shape, ask other contributors using [@mentions](https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax#mentioning-people-and-teams). @@ -78,7 +78,7 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). The Nix reference manual is hosted on https://nixos.org/manual/nix. The underlying source files are located in [`doc/manual/src`](./doc/manual/src). For small changes you can [use GitHub to edit these files](https://docs.github.com/en/repositories/working-with-files/managing-files/editing-files) -For larger changes see the [Nix reference manual](https://nixos.org/manual/nix/unstable/contributing/hacking.html). +For larger changes see the [Nix reference manual](https://nix.dev/manual/nix/development/development/contributing.html). ## Getting help diff --git a/README.md b/README.md index 931a60bba..ab647e53b 100644 --- a/README.md +++ b/README.md @@ -4,19 +4,18 @@ [![Test](https://github.com/NixOS/nix/workflows/Test/badge.svg)](https://github.com/NixOS/nix/actions) Nix is a powerful package manager for Linux and other Unix systems that makes package -management reliable and reproducible. Please refer to the [Nix manual](https://nixos.org/nix/manual) +management reliable and reproducible. Please refer to the [Nix manual](https://nix.dev/reference/nix-manual) for more details. ## Installation and first steps Visit [nix.dev](https://nix.dev) for [installation instructions](https://nix.dev/tutorials/install-nix) and [beginner tutorials](https://nix.dev/tutorials/first-steps). -Full reference documentation can be found in the [Nix manual](https://nixos.org/nix/manual). +Full reference documentation can be found in the [Nix manual](https://nix.dev/reference/nix-manual). ## Building and developing -See our [Hacking guide](https://nixos.org/manual/nix/unstable/contributing/hacking.html) in our manual for instruction on how to - set up a development environment and build Nix from source. +Follow instructions in the Nix reference manual to [set up a development environment and build Nix from source](https://nix.dev/manual/nix/development/development/building.html). ## Contributing diff --git a/doc/manual/generate-builtins.nix b/doc/manual/generate-builtins.nix index 13de6c397..37ed12a43 100644 --- a/doc/manual/generate-builtins.nix +++ b/doc/manual/generate-builtins.nix @@ -12,7 +12,7 @@ let experimentalNotice = optionalString (experimental-feature != null) '' > **Note** > - > This function is only available if the [`${experimental-feature}` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-${experimental-feature}) is enabled. + > This function is only available if the [`${experimental-feature}` experimental feature](@docroot@/development/experimental-features.md#xp-feature-${experimental-feature}) is enabled. > > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): > diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 90eaa1a73..791bfd2c7 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -38,7 +38,7 @@ let result = '' > **Warning** \ > This program is - > [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) + > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) > and its interface is subject to change. # Name diff --git a/doc/manual/generate-settings.nix b/doc/manual/generate-settings.nix index 504cda362..93a8e093e 100644 --- a/doc/manual/generate-settings.nix +++ b/doc/manual/generate-settings.nix @@ -33,10 +33,10 @@ let > **Warning** > > This setting is part of an - > [experimental feature](@docroot@/contributing/experimental-features.md). + > [experimental feature](@docroot@/development/experimental-features.md). > > To change this setting, make sure the - > [`${experimentalFeature}` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}) + > [`${experimentalFeature}` experimental feature](@docroot@/development/experimental-features.md#xp-feature-${experimentalFeature}) > is enabled. > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): > diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix index c311c3c39..cc3704124 100644 --- a/doc/manual/generate-store-info.nix +++ b/doc/manual/generate-store-info.nix @@ -32,10 +32,10 @@ let > **Warning** > > This store is part of an - > [experimental feature](@docroot@/contributing/experimental-features.md). + > [experimental feature](@docroot@/development/experimental-features.md). > > To use this store, make sure the - > [`${experimentalFeature}` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}) + > [`${experimentalFeature}` experimental feature](@docroot@/development/experimental-features.md#xp-feature-${experimentalFeature}) > is enabled. > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): > diff --git a/doc/manual/generate-xp-features-shortlist.nix b/doc/manual/generate-xp-features-shortlist.nix index ec09f4b75..eb735ba5f 100644 --- a/doc/manual/generate-xp-features-shortlist.nix +++ b/doc/manual/generate-xp-features-shortlist.nix @@ -4,6 +4,6 @@ with import ; let showExperimentalFeature = name: doc: '' - - [`${name}`](@docroot@/contributing/experimental-features.md#xp-feature-${name}) + - [`${name}`](@docroot@/development/experimental-features.md#xp-feature-${name}) ''; in xps: indent " " (concatStrings (attrValues (mapAttrs showExperimentalFeature xps))) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 0cec52885..fcc50f460 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -95,7 +95,7 @@ $(d)/nix-profiles.5: $(d)/src/command-ref/files/profiles.md $(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@ @rm $^.tmp -$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md +$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/development/experimental-feature-descriptions.md @cp $< $@ @$(call process-includes,$@,$@) @@ -124,7 +124,7 @@ $(d)/conf-file.json: $(doc_nix) $(trace-gen) $(dummy-env) $(doc_nix) config show --json --experimental-features nix-command > $@.tmp @mv $@.tmp $@ -$(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(doc_nix) +$(d)/src/development/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(doc_nix) @rm -rf $@ $@.tmp $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features.nix (builtins.fromJSON (builtins.readFile $<))' @mv $@.tmp $@ @@ -207,11 +207,11 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli done @touch $@ -# the `! -name 'contributing.md'` filter excludes the one place where +# the `! -name 'documentation.md'` filter excludes the one place where # `@docroot@` is to be preserved for documenting the mechanism # FIXME: maybe contributing guides should live right next to the code # instead of in the manual -$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/release-notes/rl-next.md $(d)/src/figures $(d)/src/favicon.png $(d)/src/favicon.svg +$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/development/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/release-notes/rl-next.md $(d)/src/figures $(d)/src/favicon.png $(d)/src/favicon.svg $(trace-gen) \ tmp="$$(mktemp -d)"; \ cp -r doc/manual "$$tmp"; \ diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index 0f9f91b03..beef6ef4a 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -143,7 +143,7 @@ const redirects = { "opt-timeout": "command-ref/opt-common.html#opt-timeout", "sec-common-options": "command-ref/opt-common.html", "ch-utilities": "command-ref/utilities.html", - "chap-hacking": "contributing/hacking.html", + "chap-hacking": "development/building.html", "adv-attr-allowSubstitutes": "language/advanced-attributes.html#adv-attr-allowSubstitutes", "adv-attr-allowedReferences": "language/advanced-attributes.html#adv-attr-allowedReferences", "adv-attr-allowedRequisites": "language/advanced-attributes.html#adv-attr-allowedRequisites", @@ -350,7 +350,7 @@ const redirects = { "macos": "uninstall.html#macos", "uninstalling": "uninstall.html", }, - "contributing/hacking.html": { + "development/building.html": { "nix-with-flakes": "#building-nix-with-flakes", "classic-nix": "#building-nix", "running-tests": "testing.html#running-tests", @@ -361,7 +361,12 @@ const redirects = { "installer-tests": "testing.html#installer-tests", "one-time-setup": "testing.html#one-time-setup", "using-the-ci-generated-installer-for-manual-testing": "testing.html#using-the-ci-generated-installer-for-manual-testing", - "characterization-testing": "#characterisation-testing-unit", + "characterization-testing": "testing.html#characterisation-testing-unit", + "add-a-release-note": "contributing.html#add-a-release-note", + "add-an-entry": "contributing.html#add-an-entry", + "build-process": "contributing.html#build-process", + "reverting": "contributing.html#reverting", + "branches": "contributing.html#branches", }, "glossary.html": { "gloss-local-store": "store/types/local-store.html", diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index b4dd277e3..3918faeb2 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -116,14 +116,15 @@ - [Derivation "ATerm" file format](protocols/derivation-aterm.md) - [C API](c-api.md) - [Glossary](glossary.md) -- [Contributing](contributing/index.md) - - [Hacking](contributing/hacking.md) - - [Testing](contributing/testing.md) - - [Documentation](contributing/documentation.md) - - [Experimental Features](contributing/experimental-features.md) - - [CLI guideline](contributing/cli-guideline.md) - - [JSON guideline](contributing/json-guideline.md) - - [C++ style guide](contributing/cxx.md) +- [Development](development/index.md) + - [Building](development/building.md) + - [Testing](development/testing.md) + - [Documentation](development/documentation.md) + - [CLI guideline](development/cli-guideline.md) + - [JSON guideline](development/json-guideline.md) + - [C++ style guide](development/cxx.md) + - [Experimental Features](development/experimental-features.md) + - [Contributing](development/contributing.md) - [Releases](release-notes/index.md) {{#include ./SUMMARY-rl-next.md}} - [Release 2.23 (2024-06-03)](release-notes/rl-2.23.md) diff --git a/doc/manual/src/_redirects b/doc/manual/src/_redirects index 578c48f06..07b3130f9 100644 --- a/doc/manual/src/_redirects +++ b/doc/manual/src/_redirects @@ -20,7 +20,15 @@ /command-ref/command-ref /command-ref 301! -/contributing/contributing /contributing 301! +/contributing/contributing /development 301! +/contributing /development 301! +/contributing/hacking /development/building 301! +/contributing/testing /development/testing 301! +/contributing/documentation /development/documentation 301! +/contributing/experimental-features /development/experimental-features 301! +/contributing/cli-guideline /development/cli-guideline 301! +/contributing/json-guideline /development/json-guideline 301! +/contributing/cxx /development/cxx 301! /expressions/expression-language /language/ 301! /expressions/language-constructs /language/constructs 301! diff --git a/doc/manual/src/c-api.md b/doc/manual/src/c-api.md index 29df0b644..0cdd83832 100644 --- a/doc/manual/src/c-api.md +++ b/doc/manual/src/c-api.md @@ -10,7 +10,7 @@ See: - [Matrix Room *Nix Bindings*](https://matrix.to/#/#nix-bindings:nixos.org) for discussion and questions. - [Stabilisation Milestone](https://github.com/NixOS/nix/milestone/52) - [Other C API PRs and issues](https://github.com/NixOS/nix/labels/c%20api) -- [Contributing C API Documentation](contributing/documentation.md#c-api-documentation), including how to build it locally. +- [Contributing C API Documentation](development/documentation.md#c-api-documentation), including how to build it locally. [Getting Started]: https://hydra.nixos.org/job/nix/master/external-api-docs/latest/download-by-type/doc/external-api-docs [Index]: https://hydra.nixos.org/job/nix/master/external-api-docs/latest/download-by-type/doc/external-api-docs/globals.html diff --git a/doc/manual/src/command-ref/experimental-commands.md b/doc/manual/src/command-ref/experimental-commands.md index 286ddc6d6..1190729a2 100644 --- a/doc/manual/src/command-ref/experimental-commands.md +++ b/doc/manual/src/command-ref/experimental-commands.md @@ -1,6 +1,6 @@ # Experimental Commands -This section lists [experimental commands](@docroot@/contributing/experimental-features.md#xp-feature-nix-command). +This section lists [experimental commands](@docroot@/development/experimental-features.md#xp-feature-nix-command). > **Warning** > diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index e30b351a4..a899758df 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -32,7 +32,7 @@ If no substitutes are available and no store derivation is given, realisation fa [store objects]: @docroot@/store/store-object.md [closure]: @docroot@/glossary.md#gloss-closure [substituters]: @docroot@/command-ref/conf-file.md#conf-substituters -[content-addressed derivations]: @docroot@/contributing/experimental-features.md#xp-feature-ca-derivations +[content-addressed derivations]: @docroot@/development/experimental-features.md#xp-feature-ca-derivations [Nix database]: @docroot@/glossary.md#gloss-nix-database The resulting paths are printed on standard output. diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/development/building.md similarity index 74% rename from doc/manual/src/contributing/hacking.md rename to doc/manual/src/development/building.md index c128515e9..5a5fb3368 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/development/building.md @@ -1,24 +1,67 @@ -# Hacking +# Building Nix -This section provides some notes on how to hack on Nix. To get the -latest version of Nix from GitHub: +This section provides some notes on how to start hacking on Nix. +To get the latest version of Nix from GitHub: ```console $ git clone https://github.com/NixOS/nix.git $ cd nix ``` -The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. If you don't have it installed, follow the [installation instructions]. +> **Note** +> +> The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. +> If you don't have it installed, follow the [installation instructions](../installation/index.md). -[installation instructions]: ../installation/index.md + +To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: + +```console +$ nix-shell +``` + +To get a shell with one of the other [supported compilation environments](#compilation-environments): + +```console +$ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages +``` + +> **Note** +> +> You can use `native-ccacheStdenvPackages` to drastically improve rebuild time. +> By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`. + +To build Nix itself in this shell: + +```console +[nix-shell]$ autoreconfPhase +[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out +[nix-shell]$ make -j $NIX_BUILD_CORES +``` + +To install it in `$(pwd)/outputs` and test it: + +```console +[nix-shell]$ make install +[nix-shell]$ make installcheck -j $NIX_BUILD_CORES +[nix-shell]$ ./outputs/out/bin/nix --version +nix (Nix) 2.12 +``` + +To build a release version of Nix for the current operating system and CPU architecture: + +```console +$ nix-build +``` + +You can also build Nix for one of the [supported platforms](#platforms). ## Building Nix with flakes This section assumes you are using Nix with the [`flakes`] and [`nix-command`] experimental features enabled. -See the [Building Nix](#building-nix) section for equivalent instructions using stable Nix interfaces. -[`flakes`]: @docroot@/contributing/experimental-features.md#xp-feature-flakes -[`nix-command`]: @docroot@/contributing/experimental-features.md#xp-nix-command +[`flakes`]: @docroot@/development/experimental-features.md#xp-feature-flakes +[`nix-command`]: @docroot@/development/experimental-features.md#xp-nix-command To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: @@ -67,50 +110,6 @@ $ nix build You can also build Nix for one of the [supported platforms](#platforms). -## Building Nix - -To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: - -```console -$ nix-shell -``` - -To get a shell with one of the other [supported compilation environments](#compilation-environments): - -```console -$ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages -``` - -> **Note** -> -> You can use `native-ccacheStdenvPackages` to drastically improve rebuild time. -> By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`. - -To build Nix itself in this shell: - -```console -[nix-shell]$ autoreconfPhase -[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out -[nix-shell]$ make -j $NIX_BUILD_CORES -``` - -To install it in `$(pwd)/outputs` and test it: - -```console -[nix-shell]$ make install -[nix-shell]$ make installcheck -j $NIX_BUILD_CORES -[nix-shell]$ ./outputs/out/bin/nix --version -nix (Nix) 2.12 -``` - -To build a release version of Nix for the current operating system and CPU architecture: - -```console -$ nix-build -``` - -You can also build Nix for one of the [supported platforms](#platforms). - ## Makefile variables You may need `profiledir=$out/etc/profile.d` and `sysconfdir=$out/etc` to run `make install`. @@ -294,81 +293,3 @@ If it fails, run `git add --patch` to approve the suggestions _and commit again_ To refresh pre-commit hook's config file, do the following: 1. Exit the development shell and start it again by running `nix develop`. 2. If you also use the pre-commit hook, also run `pre-commit-hooks-install` again. - -## Add a release note - -`doc/manual/rl-next` contains release notes entries for all unreleased changes. - -User-visible changes should come with a release note. - -### Add an entry - -Here's what a complete entry looks like. The file name is not incorporated in the document. - -``` ---- -synopsis: Basically a title -issues: 1234 -prs: 1238 ---- - -Here's one or more paragraphs that describe the change. - -- It's markdown -- Add references to the manual using @docroot@ -``` - -Significant changes should add the following header, which moves them to the top. - -``` -significance: significant -``` - - -See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog). - -### Build process - -Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. - -## Branches - -- [`master`](https://github.com/NixOS/nix/commits/master) - - The main development branch. All changes are approved and merged here. - When developing a change, create a branch based on the latest `master`. - - Maintainers try to [keep it in a release-worthy state](#reverting). - -- [`maintenance-*.*`](https://github.com/NixOS/nix/branches/all?query=maintenance) - - These branches are the subject of backports only, and are - also [kept](#reverting) in a release-worthy state. - - See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) - -- [`latest-release`](https://github.com/NixOS/nix/tree/latest-release) - - The latest patch release of the latest minor version. - - See [`maintainers/release-process.md`](https://github.com/NixOS/nix/blob/master/maintainers/release-process.md) - -- [`backport-*-to-*`](https://github.com/NixOS/nix/branches/all?query=backport) - - Generally branches created by the backport action. - - See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) - -- [_other_](https://github.com/NixOS/nix/branches/all) - - Branches that do not conform to the above patterns should be feature branches. - -## Reverting - -If a change turns out to be merged by mistake, or contain a regression, it may be reverted. -A revert is not a rejection of the contribution, but merely part of an effective development process. -It makes sure that development keeps running smoothly, with minimal uncertainty, and less overhead. -If maintainers have to worry too much about avoiding reverts, they would not be able to merge as much. -By embracing reverts as a good part of the development process, everyone wins. - -However, taking a step back may be frustrating, so maintainers will be extra supportive on the next try. diff --git a/doc/manual/src/contributing/cli-guideline.md b/doc/manual/src/development/cli-guideline.md similarity index 100% rename from doc/manual/src/contributing/cli-guideline.md rename to doc/manual/src/development/cli-guideline.md diff --git a/doc/manual/src/development/contributing.md b/doc/manual/src/development/contributing.md new file mode 100644 index 000000000..7de7489dc --- /dev/null +++ b/doc/manual/src/development/contributing.md @@ -0,0 +1,79 @@ +# Contributing + +## Add a release note + +`doc/manual/rl-next` contains release notes entries for all unreleased changes. + +User-visible changes should come with a release note. + +### Add an entry + +Here's what a complete entry looks like. The file name is not incorporated in the document. + +``` +--- +synopsis: Basically a title +issues: 1234 +prs: 1238 +--- + +Here's one or more paragraphs that describe the change. + +- It's markdown +- Add references to the manual using @docroot@ +``` + +Significant changes should add the following header, which moves them to the top. + +``` +significance: significant +``` + + +See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog). + +### Build process + +Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. + +## Branches + +- [`master`](https://github.com/NixOS/nix/commits/master) + + The main development branch. All changes are approved and merged here. + When developing a change, create a branch based on the latest `master`. + + Maintainers try to [keep it in a release-worthy state](#reverting). + +- [`maintenance-*.*`](https://github.com/NixOS/nix/branches/all?query=maintenance) + + These branches are the subject of backports only, and are + also [kept](#reverting) in a release-worthy state. + + See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) + +- [`latest-release`](https://github.com/NixOS/nix/tree/latest-release) + + The latest patch release of the latest minor version. + + See [`maintainers/release-process.md`](https://github.com/NixOS/nix/blob/master/maintainers/release-process.md) + +- [`backport-*-to-*`](https://github.com/NixOS/nix/branches/all?query=backport) + + Generally branches created by the backport action. + + See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) + +- [_other_](https://github.com/NixOS/nix/branches/all) + + Branches that do not conform to the above patterns should be feature branches. + +## Reverting + +If a change turns out to be merged by mistake, or contain a regression, it may be reverted. +A revert is not a rejection of the contribution, but merely part of an effective development process. +It makes sure that development keeps running smoothly, with minimal uncertainty, and less overhead. +If maintainers have to worry too much about avoiding reverts, they would not be able to merge as much. +By embracing reverts as a good part of the development process, everyone wins. + +However, taking a step back may be frustrating, so maintainers will be extra supportive on the next try. diff --git a/doc/manual/src/contributing/cxx.md b/doc/manual/src/development/cxx.md similarity index 100% rename from doc/manual/src/contributing/cxx.md rename to doc/manual/src/development/cxx.md diff --git a/doc/manual/src/contributing/documentation.md b/doc/manual/src/development/documentation.md similarity index 99% rename from doc/manual/src/contributing/documentation.md rename to doc/manual/src/development/documentation.md index a14ecedd6..63f574ab7 100644 --- a/doc/manual/src/contributing/documentation.md +++ b/doc/manual/src/development/documentation.md @@ -24,7 +24,7 @@ nix build .#^doc and open `./result-doc/share/doc/nix/manual/index.html`. -To build the manual incrementally, [enter the development shell](./hacking.md) and run: +To build the manual incrementally, [enter the development shell](./building.md) and run: ```console make manual-html-open -j $NIX_BUILD_CORES diff --git a/doc/manual/src/contributing/experimental-features.md b/doc/manual/src/development/experimental-features.md similarity index 100% rename from doc/manual/src/contributing/experimental-features.md rename to doc/manual/src/development/experimental-features.md diff --git a/doc/manual/src/contributing/index.md b/doc/manual/src/development/index.md similarity index 77% rename from doc/manual/src/contributing/index.md rename to doc/manual/src/development/index.md index 4d55c17a4..6403c3e66 100644 --- a/doc/manual/src/contributing/index.md +++ b/doc/manual/src/development/index.md @@ -5,4 +5,4 @@ Check the [contributing guide](https://github.com/NixOS/nix/blob/master/CONTRIBU This chapter is a collection of guides for making changes to the code and documentation. -If you're not sure where to start, try to [compile Nix from source](./hacking.md) and consider [making improvements to documentation](./documentation.md). +If you're not sure where to start, try to [compile Nix from source](./building.md) and consider [making improvements to documentation](./documentation.md). diff --git a/doc/manual/src/contributing/json-guideline.md b/doc/manual/src/development/json-guideline.md similarity index 100% rename from doc/manual/src/contributing/json-guideline.md rename to doc/manual/src/development/json-guideline.md diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/development/testing.md similarity index 100% rename from doc/manual/src/contributing/testing.md rename to doc/manual/src/development/testing.md diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index f65ada63a..877c4668b 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -168,7 +168,7 @@ - [impure derivation]{#gloss-impure-derivation} - [An experimental feature](#@docroot@/contributing/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, + [An experimental feature](#@docroot@/development/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, so that they are always rebuilt, and their outputs not reused by subsequent calls to realise them. - [Nix database]{#gloss-nix-database} @@ -353,7 +353,7 @@ Not yet stabilized functionality guarded by named experimental feature flags. These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting. - See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/contributing/experimental-features.md). + See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/development/experimental-features.md). [Nix language]: ./language/index.md diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index 385008d8c..6a168ff3d 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -77,7 +77,7 @@ $ su root # Installing from a binary tarball You can also download a binary tarball that contains Nix and all its dependencies: -- Choose a [version](https://releases.nixos.org/?prefix=nix/) and [system type](../contributing/hacking.md#platforms) +- Choose a [version](https://releases.nixos.org/?prefix=nix/) and [system type](../development/building.md#platforms) - Download and unpack the tarball - Run the installer diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index e916c7360..51b83fc8a 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -113,7 +113,7 @@ Derivations can declare some infrequently used optional attributes. > `nix-build`. If the [`configurable-impure-env` experimental - feature](@docroot@/contributing/experimental-features.md#xp-feature-configurable-impure-env) + feature](@docroot@/development/experimental-features.md#xp-feature-configurable-impure-env) is enabled, these environment variables can also be controlled through the [`impure-env`](@docroot@/command-ref/conf-file.md#conf-impure-env) @@ -226,7 +226,7 @@ Derivations can declare some infrequently used optional attributes. - [`__contentAddressed`]{#adv-attr-__contentAddressed} > **Warning** - > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md). + > This attribute is part of an [experimental feature](@docroot@/development/experimental-features.md). > > To use this attribute, you must enable the > [`ca-derivations`][xp-feature-ca-derivations] experimental feature. @@ -370,6 +370,6 @@ Derivations can declare some infrequently used optional attributes. ensures that the derivation can only be built on a machine with the `kvm` feature. -[xp-feature-ca-derivations]: @docroot@/contributing/experimental-features.md#xp-feature-ca-derivations -[xp-feature-dynamic-derivations]: @docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations -[xp-feature-git-hashing]: @docroot@/contributing/experimental-features.md#xp-feature-git-hashing +[xp-feature-ca-derivations]: @docroot@/development/experimental-features.md#xp-feature-ca-derivations +[xp-feature-dynamic-derivations]: @docroot@/development/experimental-features.md#xp-feature-dynamic-derivations +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/protocols/derivation-aterm.md b/doc/manual/src/protocols/derivation-aterm.md index e58b602a3..1ba757ae0 100644 --- a/doc/manual/src/protocols/derivation-aterm.md +++ b/doc/manual/src/protocols/derivation-aterm.md @@ -14,6 +14,6 @@ Derivations are serialised in one of the following formats: DrvWithVersion(, ...) ``` - The only `version-string`s that are in use today are for [experimental features](@docroot@/contributing/experimental-features.md): + The only `version-string`s that are in use today are for [experimental features](@docroot@/development/experimental-features.md): - - `"xp-dyn-drv"` for the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. + - `"xp-dyn-drv"` for the [`dynamic-derivations`](@docroot@/development/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. diff --git a/doc/manual/src/protocols/json/derivation.md b/doc/manual/src/protocols/json/derivation.md index f881dd703..2f85340d6 100644 --- a/doc/manual/src/protocols/json/derivation.md +++ b/doc/manual/src/protocols/json/derivation.md @@ -3,7 +3,7 @@ > **Warning** > > This JSON format is currently -> [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) +> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) > and subject to change. The JSON serialization of a diff --git a/doc/manual/src/protocols/json/store-object-info.md b/doc/manual/src/protocols/json/store-object-info.md index 9f647a96c..6b4f48437 100644 --- a/doc/manual/src/protocols/json/store-object-info.md +++ b/doc/manual/src/protocols/json/store-object-info.md @@ -3,7 +3,7 @@ > **Warning** > > This JSON format is currently -> [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) +> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) > and subject to change. Info about a [store object]. diff --git a/doc/manual/src/release-notes/rl-2.18.md b/doc/manual/src/release-notes/rl-2.18.md index 4bbc52b50..eb26fc9e7 100644 --- a/doc/manual/src/release-notes/rl-2.18.md +++ b/doc/manual/src/release-notes/rl-2.18.md @@ -13,7 +13,7 @@ - The `discard-references` feature has been stabilized. This means that the - [unsafeDiscardReferences](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) + [unsafeDiscardReferences](@docroot@/development/experimental-features.md#xp-feature-discard-references) attribute is no longer guarded by an experimental flag and can be used freely. @@ -21,7 +21,7 @@ This only affects `nix-build --json` when "building" non-derivation things like fetched sources, which is a no-op. - A new builtin [`outputOf`](@docroot@/language/builtins.md#builtins-outputOf) has been added. - It is part of the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. + It is part of the [`dynamic-derivations`](@docroot@/development/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. - Flake follow paths at depths greater than 2 are now handled correctly, preventing "follows a non-existent input" errors. diff --git a/doc/manual/src/release-notes/rl-2.19.md b/doc/manual/src/release-notes/rl-2.19.md index ba6eb9c64..e2e2f85cc 100644 --- a/doc/manual/src/release-notes/rl-2.19.md +++ b/doc/manual/src/release-notes/rl-2.19.md @@ -17,8 +17,8 @@ - `nix-shell` shebang lines now support single-quoted arguments. -- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). - This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). +- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/development/experimental-features.md#xp-fetch-tree). + This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/development/experimental-features.md#xp-fetch-tree). - The interface for creating and updating lock files has been overhauled: @@ -33,7 +33,7 @@ - The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables. They are superceded by `nix flake update`. -- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). +- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). - [`nix path-info --json`](@docroot@/command-ref/new-cli/nix3-path-info.md) (experimental) now returns a JSON map rather than JSON list. diff --git a/doc/manual/src/release-notes/rl-2.23.md b/doc/manual/src/release-notes/rl-2.23.md index 3c59b8583..ac842fdc0 100644 --- a/doc/manual/src/release-notes/rl-2.23.md +++ b/doc/manual/src/release-notes/rl-2.23.md @@ -14,7 +14,7 @@ - Modify `nix derivation {add,show}` JSON format [#9866](https://github.com/NixOS/nix/issues/9866) [#10722](https://github.com/NixOS/nix/pull/10722) - The JSON format for derivations has been slightly revised to better conform to our [JSON guidelines](@docroot@/contributing/cli-guideline.md#returning-future-proof-json). + The JSON format for derivations has been slightly revised to better conform to our [JSON guidelines](@docroot@/development/cli-guideline.md#returning-future-proof-json). In particular, the hash algorithm and content addressing method of content-addresed derivation outputs are now separated into two fields `hashAlgo` and `method`, rather than one field with an arcane `:`-separated format. @@ -89,7 +89,7 @@ This makes records of this sort more self-describing, and easier to consume programmatically. We will follow this design principle going forward; - the [JSON guidelines](@docroot@/contributing/json-guideline.md) in the contributing section have been updated accordingly. + the [JSON guidelines](@docroot@/development/json-guideline.md) in the contributing section have been updated accordingly. - Large path warnings [#10661](https://github.com/NixOS/nix/pull/10661) diff --git a/doc/manual/src/release-notes/rl-2.4.md b/doc/manual/src/release-notes/rl-2.4.md index 8b566fc7b..1201e53b6 100644 --- a/doc/manual/src/release-notes/rl-2.4.md +++ b/doc/manual/src/release-notes/rl-2.4.md @@ -23,7 +23,7 @@ more than 2800 commits from 195 contributors since release 2.3. * The **`nix` command** has seen a lot of work and is now almost at feature parity with the old command-line interface (the `nix-*` commands). It aims to be [more modern, consistent and pleasant to - use](../contributing/cli-guideline.md) than the old CLI. It is still + use](../development/cli-guideline.md) than the old CLI. It is still marked as experimental but its interface should not change much anymore in future releases. diff --git a/doc/manual/src/store/file-system-object/content-address.md b/doc/manual/src/store/file-system-object/content-address.md index 1c63c52eb..410d7fb7c 100644 --- a/doc/manual/src/store/file-system-object/content-address.md +++ b/doc/manual/src/store/file-system-object/content-address.md @@ -82,4 +82,4 @@ In the future, we may support a Git-like hash for such file system objects, or w [file system object]: ../file-system-object.md [store object]: ../store-object.md -[xp-feature-git-hashing]: @docroot@/contributing/experimental-features.md#xp-feature-git-hashing +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/store/store-object/content-address.md b/doc/manual/src/store/store-object/content-address.md index f6f982035..02dce2836 100644 --- a/doc/manual/src/store/store-object/content-address.md +++ b/doc/manual/src/store/store-object/content-address.md @@ -92,4 +92,4 @@ becomes more widespread, this restriction will be revisited. [fso-ca]: ../file-system-object/content-address.md [sp-spec]: @docroot@/protocols/store-path.md -[xp-feature-git-hashing]: @docroot@/contributing/experimental-features.md#xp-feature-git-hashing +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index 8f48b53a5..30a8c5c58 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -65,7 +65,7 @@ struct EvalSettings : Config extern "C" typedef void (*ValueInitialiser) (EvalState & state, Value & v); ``` - The [Nix C++ API documentation](@docroot@/contributing/documentation.md#api-documentation) has more details on evaluator internals. + The [Nix C++ API documentation](@docroot@/development/documentation.md#api-documentation) has more details on evaluator internals. - `builtins.exec` *arguments* diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 333e486fd..e59d7fe67 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -383,7 +383,7 @@ static RegisterPrimOp primop_fetchTree({ - `"mercurial"` *input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references). - The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled. + The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/development/experimental-features.md#xp-feature-flakes) to be enabled. > **Example** > @@ -670,12 +670,12 @@ static RegisterPrimOp primop_fetchGit({ Whether to check `rev` for a signature matching `publicKey` or `publicKeys`. If `verifyCommit` is enabled, then `fetchGit` cannot use a local repository with uncommitted changes. - Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + Requires the [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). - `publicKey` The public key against which `rev` is verified if `verifyCommit` is enabled. - Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + Requires the [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). - `keytype` (default: `"ssh-ed25519"`) @@ -687,7 +687,7 @@ static RegisterPrimOp primop_fetchGit({ - `"ssh-ed25519"` - `"ssh-ed25519-sk"` - `"ssh-rsa"` - Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + Requires the [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). - `publicKeys` @@ -701,7 +701,7 @@ static RegisterPrimOp primop_fetchGit({ } ``` - Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + Requires the [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). Here are some examples of how to use `fetchGit`. diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 30d7537bd..8760c9d14 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -286,7 +286,7 @@ public: For backward compatibility, `ssh://` may be omitted. The hostname may be an alias defined in `~/.ssh/config`. - 2. A comma-separated list of [Nix system types](@docroot@/contributing/hacking.md#system-type). + 2. A comma-separated list of [Nix system types](@docroot@/development/building.md#system-type). If omitted, this defaults to the local platform type. > **Example** @@ -866,13 +866,13 @@ public: - `ca-derivations` - Included by default if the [`ca-derivations` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) is enabled. + Included by default if the [`ca-derivations` experimental feature](@docroot@/development/experimental-features.md#xp-feature-ca-derivations) is enabled. This system feature is implicitly required by derivations with the [`__contentAddressed` attribute](@docroot@/language/advanced-attributes.md#adv-attr-__contentAddressed). - `recursive-nix` - Included by default if the [`recursive-nix` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-recursive-nix) is enabled. + Included by default if the [`recursive-nix` experimental feature](@docroot@/development/experimental-features.md#xp-feature-recursive-nix) is enabled. - `uid-range` diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 1952ba1b8..c0c59ac68 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -393,7 +393,7 @@ struct ExperimentalFeatureSettings : Config { {{#include experimental-features-shortlist.md}} - Experimental features are [further documented in the manual](@docroot@/contributing/experimental-features.md). + Experimental features are [further documented in the manual](@docroot@/development/experimental-features.md). )"}; /** diff --git a/src/nix/nix.md b/src/nix/nix.md index 4464bef37..f958ce09a 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -50,7 +50,7 @@ manual](https://nixos.org/manual/nix/stable/). > **Warning** \ > Installables are part of the unstable -> [`nix-command` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-nix-command), +> [`nix-command` experimental feature](@docroot@/development/experimental-features.md#xp-feature-nix-command), > and subject to change without notice. Many `nix` subcommands operate on one or more *installables*. @@ -70,9 +70,9 @@ That is, Nix will operate on the default flake output attribute of the flake in > **Warning** \ > Flake output attribute installables depend on both the -> [`flakes`](@docroot@/contributing/experimental-features.md#xp-feature-flakes) +> [`flakes`](@docroot@/development/experimental-features.md#xp-feature-flakes) > and -> [`nix-command`](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) +> [`nix-command`](@docroot@/development/experimental-features.md#xp-feature-nix-command) > experimental features, and subject to change without notice. Example: `nixpkgs#hello` From 3b49f7a1439bc75cefddb0fe0031824f3708da83 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 24 Jul 2024 23:12:39 -0400 Subject: [PATCH 043/142] Deduplicate our many `package.nix` a bit (#11175) - They should all be built in parallel - They should all use strict deps by default --- packaging/dependencies.nix | 27 ++++++++++++++++++------- src/external-api-docs/package.nix | 4 ---- src/internal-api-docs/package.nix | 4 ---- src/libcmd/package.nix | 6 ------ src/libexpr-c/package.nix | 4 ---- src/libexpr/package.nix | 4 ---- src/libfetchers/package.nix | 6 ------ src/libflake/package.nix | 6 ------ src/libmain-c/package.nix | 4 ---- src/libmain/package.nix | 6 ------ src/libstore-c/package.nix | 4 ---- src/libstore/package.nix | 4 ---- src/libutil-c/package.nix | 4 ---- src/libutil/package.nix | 4 ---- src/nix/package.nix | 6 ------ src/perl/package.nix | 2 +- tests/unit/libexpr-support/package.nix | 4 ---- tests/unit/libexpr/package.nix | 4 ---- tests/unit/libfetchers/package.nix | 4 ---- tests/unit/libflake/package.nix | 4 ---- tests/unit/libstore-support/package.nix | 4 ---- tests/unit/libstore/package.nix | 4 ---- tests/unit/libutil-support/package.nix | 4 ---- tests/unit/libutil/package.nix | 4 ---- 24 files changed, 21 insertions(+), 106 deletions(-) diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index f09ca5d18..b77e1d14f 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -63,9 +63,16 @@ let # Work around weird `--as-needed` linker behavior with BSD, see # https://github.com/mesonbuild/meson/issues/3593 - bsdNoLinkAsNeeded = finalAttrs: prevAttrs: lib.optionalAttrs stdenv.hostPlatform.isBSD { - mesonFlags = [ (lib.mesonBool "b_asneeded" false) ] ++ prevAttrs.mesonFlags or []; - }; + bsdNoLinkAsNeeded = finalAttrs: prevAttrs: + lib.optionalAttrs stdenv.hostPlatform.isBSD { + mesonFlags = [ (lib.mesonBool "b_asneeded" false) ] ++ prevAttrs.mesonFlags or []; + }; + + miscGoodPractice = finalAttrs: prevAttrs: + { + strictDeps = prevAttrs.strictDeps or true; + enableParallelBuilding = true; + }; in scope: { @@ -136,8 +143,14 @@ scope: { inherit resolvePath filesetToSource; - mkMesonDerivation = f: stdenv.mkDerivation - (lib.extends - (lib.composeExtensions bsdNoLinkAsNeeded localSourceLayer) - f); + mkMesonDerivation = f: let + exts = [ + miscGoodPractice + bsdNoLinkAsNeeded + localSourceLayer + ]; + in stdenv.mkDerivation + (lib.extends + (lib.foldr lib.composeExtensions (_: _: {}) exts) + f); } diff --git a/src/external-api-docs/package.nix b/src/external-api-docs/package.nix index da136bbe1..743b3e9b7 100644 --- a/src/external-api-docs/package.nix +++ b/src/external-api-docs/package.nix @@ -53,10 +53,6 @@ mkMesonDerivation (finalAttrs: { echo "doc external-api-docs $out/share/doc/nix/external-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products ''; - enableParallelBuilding = true; - - strictDeps = true; - meta = { platforms = lib.platforms.all; }; diff --git a/src/internal-api-docs/package.nix b/src/internal-api-docs/package.nix index f2077dcaf..07ca6d4d9 100644 --- a/src/internal-api-docs/package.nix +++ b/src/internal-api-docs/package.nix @@ -48,10 +48,6 @@ mkMesonDerivation (finalAttrs: { echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products ''; - enableParallelBuilding = true; - - strictDeps = true; - meta = { platforms = lib.platforms.all; }; diff --git a/src/libcmd/package.nix b/src/libcmd/package.nix index ec3aa4660..cde494901 100644 --- a/src/libcmd/package.nix +++ b/src/libcmd/package.nix @@ -93,14 +93,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated - # to work with `strictDeps`. - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libexpr-c/package.nix b/src/libexpr-c/package.nix index 0b895437b..eb42195a4 100644 --- a/src/libexpr-c/package.nix +++ b/src/libexpr-c/package.nix @@ -63,12 +63,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libexpr/package.nix b/src/libexpr/package.nix index 704456c96..4d10079ff 100644 --- a/src/libexpr/package.nix +++ b/src/libexpr/package.nix @@ -102,12 +102,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libfetchers/package.nix b/src/libfetchers/package.nix index b4abb144b..9b5d8bff7 100644 --- a/src/libfetchers/package.nix +++ b/src/libfetchers/package.nix @@ -67,14 +67,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated - # to work with `strictDeps`. - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libflake/package.nix b/src/libflake/package.nix index af6f5da94..851adf07e 100644 --- a/src/libflake/package.nix +++ b/src/libflake/package.nix @@ -67,14 +67,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated - # to work with `strictDeps`. - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libmain-c/package.nix b/src/libmain-c/package.nix index 478e34a85..ce6f67300 100644 --- a/src/libmain-c/package.nix +++ b/src/libmain-c/package.nix @@ -68,12 +68,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libmain/package.nix b/src/libmain/package.nix index bbd97ec3e..47513dbdc 100644 --- a/src/libmain/package.nix +++ b/src/libmain/package.nix @@ -62,14 +62,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated - # to work with `strictDeps`. - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libstore-c/package.nix b/src/libstore-c/package.nix index fc34c1bda..e4f372236 100644 --- a/src/libstore-c/package.nix +++ b/src/libstore-c/package.nix @@ -64,12 +64,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libstore/package.nix b/src/libstore/package.nix index 0a2ace91e..02ff4194a 100644 --- a/src/libstore/package.nix +++ b/src/libstore/package.nix @@ -101,12 +101,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libutil-c/package.nix b/src/libutil-c/package.nix index 53451998d..ccfafd4d3 100644 --- a/src/libutil-c/package.nix +++ b/src/libutil-c/package.nix @@ -62,12 +62,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libutil/package.nix b/src/libutil/package.nix index 28d7d8f0e..4ce1a75b0 100644 --- a/src/libutil/package.nix +++ b/src/libutil/package.nix @@ -88,12 +88,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/nix/package.nix b/src/nix/package.nix index fe83c6969..ef7265458 100644 --- a/src/nix/package.nix +++ b/src/nix/package.nix @@ -84,8 +84,6 @@ mkMesonDerivation (finalAttrs: { ] ); - outputs = [ "out" "dev" ]; - nativeBuildInputs = [ meson ninja @@ -114,12 +112,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/perl/package.nix b/src/perl/package.nix index 26856e631..0b9343fba 100644 --- a/src/perl/package.nix +++ b/src/perl/package.nix @@ -73,5 +73,5 @@ perl.pkgs.toPerlModule (mkMesonDerivation (finalAttrs: { "--print-errorlogs" ]; - enableParallelBuilding = true; + strictDeps = false; })) diff --git a/tests/unit/libexpr-support/package.nix b/tests/unit/libexpr-support/package.nix index 0c966c55a..f53aa842f 100644 --- a/tests/unit/libexpr-support/package.nix +++ b/tests/unit/libexpr-support/package.nix @@ -66,12 +66,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/tests/unit/libexpr/package.nix b/tests/unit/libexpr/package.nix index 015e3fbc6..e70ed7836 100644 --- a/tests/unit/libexpr/package.nix +++ b/tests/unit/libexpr/package.nix @@ -71,12 +71,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; passthru = { diff --git a/tests/unit/libfetchers/package.nix b/tests/unit/libfetchers/package.nix index cf75f68e5..ad512f562 100644 --- a/tests/unit/libfetchers/package.nix +++ b/tests/unit/libfetchers/package.nix @@ -69,12 +69,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; passthru = { diff --git a/tests/unit/libflake/package.nix b/tests/unit/libflake/package.nix index d2c9fdb89..0d63d2ff7 100644 --- a/tests/unit/libflake/package.nix +++ b/tests/unit/libflake/package.nix @@ -69,12 +69,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; passthru = { diff --git a/tests/unit/libstore-support/package.nix b/tests/unit/libstore-support/package.nix index cb15cdd5f..f512db3ee 100644 --- a/tests/unit/libstore-support/package.nix +++ b/tests/unit/libstore-support/package.nix @@ -66,12 +66,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/tests/unit/libstore/package.nix b/tests/unit/libstore/package.nix index 39bf77585..7560a5b79 100644 --- a/tests/unit/libstore/package.nix +++ b/tests/unit/libstore/package.nix @@ -73,12 +73,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; passthru = { diff --git a/tests/unit/libutil-support/package.nix b/tests/unit/libutil-support/package.nix index fdecdec72..1665804cb 100644 --- a/tests/unit/libutil-support/package.nix +++ b/tests/unit/libutil-support/package.nix @@ -64,12 +64,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/tests/unit/libutil/package.nix b/tests/unit/libutil/package.nix index c7827e74f..2fce5bfa8 100644 --- a/tests/unit/libutil/package.nix +++ b/tests/unit/libutil/package.nix @@ -70,12 +70,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; passthru = { From 8a7e31362ad0b232f4de098573370f6994834397 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 25 Jul 2024 05:57:06 +0200 Subject: [PATCH 044/142] rl-next: Add credit --- doc/manual/rl-next/drop-vendored-toml11.md | 2 ++ doc/manual/rl-next/harden-user-sandboxing.md | 5 +++++ doc/manual/rl-next/nix-shell-looks-for-shell-nix.md | 2 ++ doc/manual/rl-next/repl-doc-renders-doc-comments.md | 4 +++- doc/manual/rl-next/shebang-relative.md | 2 ++ 5 files changed, 14 insertions(+), 1 deletion(-) diff --git a/doc/manual/rl-next/drop-vendored-toml11.md b/doc/manual/rl-next/drop-vendored-toml11.md index d1feeb703..8dd786c44 100644 --- a/doc/manual/rl-next/drop-vendored-toml11.md +++ b/doc/manual/rl-next/drop-vendored-toml11.md @@ -4,3 +4,5 @@ synopsis: Stop vendoring toml11 We don't apply any patches to it, and vendoring it locks users into bugs (it hasn't been updated since its introduction in late 2021). + +Author: [**Winter (@winterqt)**](https://github.com/winterqt) diff --git a/doc/manual/rl-next/harden-user-sandboxing.md b/doc/manual/rl-next/harden-user-sandboxing.md index a647acf25..ff81c9cb1 100644 --- a/doc/manual/rl-next/harden-user-sandboxing.md +++ b/doc/manual/rl-next/harden-user-sandboxing.md @@ -5,3 +5,8 @@ issues: --- The build directory has been hardened against interference with the outside world by nesting it inside another directory owned by (and only readable by) the daemon user. + +This is a low severity security fix, [CVE-2024-38531](https://www.cve.org/CVERecord?id=CVE-2024-38531), that was handled through the GitHub Security Advisories interface, and hence was merged directly in commit [2dd7f8f42](https://github.com/NixOS/nix/commit/2dd7f8f42da374d9fee4d424c1c6f82bcb36b393) instead of a PR. + +Credit: [**@alois31**](https://github.com/alois31), [**Linus Heckemann (@lheckemann)**](https://github.com/lheckemann) +Co-authors: [**@edolstra**](https://github.com/edolstra) diff --git a/doc/manual/rl-next/nix-shell-looks-for-shell-nix.md b/doc/manual/rl-next/nix-shell-looks-for-shell-nix.md index 99be4148b..b9e4b3fb3 100644 --- a/doc/manual/rl-next/nix-shell-looks-for-shell-nix.md +++ b/doc/manual/rl-next/nix-shell-looks-for-shell-nix.md @@ -26,3 +26,5 @@ This also applies to `nix-shell` shebang scripts. Consider the following example This will now load `shell.nix` from the script's directory, if it exists; `default.nix` otherwise. The old behavior can be opted into by setting the option [`nix-shell-always-looks-for-shell-nix`](@docroot@/command-ref/conf-file.md#conf-nix-shell-always-looks-for-shell-nix) to `false`. + +Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) diff --git a/doc/manual/rl-next/repl-doc-renders-doc-comments.md b/doc/manual/rl-next/repl-doc-renders-doc-comments.md index 05023697c..fa241ebc1 100644 --- a/doc/manual/rl-next/repl-doc-renders-doc-comments.md +++ b/doc/manual/rl-next/repl-doc-renders-doc-comments.md @@ -48,6 +48,8 @@ Known limitations: - It does not render documentation for "formals", such as `{ /** the value to return */ x, ... }: x`. - Some extensions to markdown are not yet supported, as you can see in the example above. -We'd like to acknowledge Yingchi Long for proposing a proof of concept for this functionality in [#9054](https://github.com/NixOS/nix/pull/9054), as well as @sternenseemann and Johannes Kirschbauer for their contributions, proposals, and their work on [RFC 145]. +We'd like to acknowledge [Yingchi Long (@inclyc)](https://github.com/inclyc) for proposing a proof of concept for this functionality in [#9054](https://github.com/NixOS/nix/pull/9054), as well as [@sternenseemann](https://github.com/sternenseemann) and [Johannes Kirschbauer (@hsjobeki)](https://github.com/hsjobeki) for their contributions, proposals, and their work on [RFC 145]. + +Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) [RFC 145]: https://github.com/NixOS/rfcs/pull/145 diff --git a/doc/manual/rl-next/shebang-relative.md b/doc/manual/rl-next/shebang-relative.md index d12c0f8dc..dd96bf203 100644 --- a/doc/manual/rl-next/shebang-relative.md +++ b/doc/manual/rl-next/shebang-relative.md @@ -60,3 +60,5 @@ Example: #!nix -c bash hello ``` + +Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) From 7275d68d3b28115e2e2096aef7e6025292d4d58e Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 25 Jul 2024 05:57:53 +0200 Subject: [PATCH 045/142] rl-next: Add top 10 by +1 reactions on PRs We should use a metric that weighs the related issues. Counterbalancing time doesn't make much sense to me. If it's around for longer, the fix will be relevant to more people. --- .../10564-attrcursor-remove-forceerrors.md | 9 ++++++ ...03-run-the-flake-regressions-test-suite.md | 8 +++++ ...unit-prefixes-in-configuration-settings.md | 10 ++++++ ...ild-show-all-fod-errors-with-keep-going.md | 10 ++++++ doc/manual/rl-next/10855-meson.md | 31 +++++++++++++++++++ .../11086-eval-cache-fix-cache-regressions.md | 14 +++++++++ .../rl-next/9063-introduce-libnixflake.md | 12 +++++++ 7 files changed, 94 insertions(+) create mode 100644 doc/manual/rl-next/10564-attrcursor-remove-forceerrors.md create mode 100644 doc/manual/rl-next/10603-run-the-flake-regressions-test-suite.md create mode 100644 doc/manual/rl-next/10668-support-unit-prefixes-in-configuration-settings.md create mode 100644 doc/manual/rl-next/10734-nix3-build-show-all-fod-errors-with-keep-going.md create mode 100644 doc/manual/rl-next/10855-meson.md create mode 100644 doc/manual/rl-next/11086-eval-cache-fix-cache-regressions.md create mode 100644 doc/manual/rl-next/9063-introduce-libnixflake.md diff --git a/doc/manual/rl-next/10564-attrcursor-remove-forceerrors.md b/doc/manual/rl-next/10564-attrcursor-remove-forceerrors.md new file mode 100644 index 000000000..864a55b51 --- /dev/null +++ b/doc/manual/rl-next/10564-attrcursor-remove-forceerrors.md @@ -0,0 +1,9 @@ +--- +synopsis: "Solve `cached failure of attribute X`" +prs: 10564 +issues: 10513 9165 +--- + +This eliminates all "cached failure of attribute X" messages by forcing evaluation of the original value when needed to show the exception to the user. This enhancement improves error reporting by providing the underlying message and stack trace. + +Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) diff --git a/doc/manual/rl-next/10603-run-the-flake-regressions-test-suite.md b/doc/manual/rl-next/10603-run-the-flake-regressions-test-suite.md new file mode 100644 index 000000000..42864323c --- /dev/null +++ b/doc/manual/rl-next/10603-run-the-flake-regressions-test-suite.md @@ -0,0 +1,8 @@ +--- +synopsis: "Run the flake regressions test suite" +prs: 10603 +--- + +This update introduces a GitHub action to run a subset of the [flake regressions test suite](https://github.com/NixOS/flake-regressions), which includes 259 flakes with their expected evaluation results. Currently, the action runs the first 25 flakes due to the full test suite's extensive runtime. A manually triggered action may be implemented later to run the entire test suite. + +Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) diff --git a/doc/manual/rl-next/10668-support-unit-prefixes-in-configuration-settings.md b/doc/manual/rl-next/10668-support-unit-prefixes-in-configuration-settings.md new file mode 100644 index 000000000..2caca9a81 --- /dev/null +++ b/doc/manual/rl-next/10668-support-unit-prefixes-in-configuration-settings.md @@ -0,0 +1,10 @@ +--- +synopsis: "Support unit prefixes in configuration settings" +prs: 10668 +--- + +Configuration settings in Nix now support unit prefixes, allowing for more intuitive and readable configurations. For example, you can now specify [`--min-free 1G`](@docroot@/command-ref/opt-common.md#opt-min-free) to set the minimum free space to 1 gigabyte. + +This enhancement was extracted from [#7851](https://github.com/NixOS/nix/pull/7851) and is also useful for PR [#10661](https://github.com/NixOS/nix/pull/10661). + +Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) diff --git a/doc/manual/rl-next/10734-nix3-build-show-all-fod-errors-with-keep-going.md b/doc/manual/rl-next/10734-nix3-build-show-all-fod-errors-with-keep-going.md new file mode 100644 index 000000000..5c2797be8 --- /dev/null +++ b/doc/manual/rl-next/10734-nix3-build-show-all-fod-errors-with-keep-going.md @@ -0,0 +1,10 @@ +--- +synopsis: "nix3-build: show all FOD errors with `--keep-going`" +prs: 10734 +--- + +The [`nix build`](@docroot@/command-ref/new-cli/nix3-build.md) command has been updated to improve the behavior of the [`--keep-going`] flag. Now, when `--keep-going` is used, all hash-mismatch errors of failing fixed-output derivations (FODs) are displayed, similar to the behavior of `nix build`. This enhancement ensures that all relevant build errors are shown, making it easier for users to update multiple derivations at once or to diagnose and fix issues. + +Author: [**Jörg Thalheim (@Mic92)**](https://github.com/Mic92) + +[`--keep-going`](@docroot@/command-ref/opt-common.md#opt-keep-going) diff --git a/doc/manual/rl-next/10855-meson.md b/doc/manual/rl-next/10855-meson.md new file mode 100644 index 000000000..0ab71390f --- /dev/null +++ b/doc/manual/rl-next/10855-meson.md @@ -0,0 +1,31 @@ +--- +synopsis: "Build with Meson" +prs: +- 10378 +- 10855 +- 10904 +- 10908 +- 10914 +- 10933 +- 10936 +- 10954 +- 10955 +- 10967 +- 10963 +- 10973 +- 11034 +- 11054 +- 11055 +- 11064 +- 11060 +- 11155 +issues: +- 2503 +--- + +These changes aim to replace the use of autotools and make with Meson for building various components of Nix. Additionally, each library is built in its own derivation, leveraging Meson's "subprojects" feature to allow a single development shell for building all libraries while also supporting separate builds. This approach aims to improve productivity and build modularity, compared to both make and a monolithic Meson-based derivation. + +Special thanks to everyone who has contributed to the Meson port, particularly [**@p01arst0rm**](https://github.com/p01arst0rm) and [**@Qyriad**](https://github.com/Qyriad). + +Authors: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314), [**Tom Bereknyei**](https://github.com/tomberek), [**Théophane Hufschmitt (@thufschmitt)**](https://github.com/thufschmitt), [**Valentin Gagarin (@fricklerhandwerk)**](https://github.com/fricklerhandwerk), [**Robert Hensing (@roberth)**](https://github.com/roberth) +Co-authors: [**@p01arst0rm**](https://github.com/p01arst0rm), [**@Qyriad**](https://github.com/Qyriad) diff --git a/doc/manual/rl-next/11086-eval-cache-fix-cache-regressions.md b/doc/manual/rl-next/11086-eval-cache-fix-cache-regressions.md new file mode 100644 index 000000000..8a348a9ad --- /dev/null +++ b/doc/manual/rl-next/11086-eval-cache-fix-cache-regressions.md @@ -0,0 +1,14 @@ +--- +synopsis: "Eval cache: fix cache regressions" +prs: 11086 +issues: 10570 +--- + +This update addresses two bugs in the evaluation cache system: + +1. Regression in #10570: The evaluation cache was not being persisted in `nix develop` because `evalCaches` retained references to the caches and was never freed. +2. Nix could sometimes try to commit the evaluation cache SQLite transaction without there being an active transaction, resulting in non-error errors being printed. + +These bug fixes ensure that the evaluation cache is correctly managed and errors are appropriately handled. + +Author: [**Lexi Mattick (@kognise)**](https://github.com/kognise) diff --git a/doc/manual/rl-next/9063-introduce-libnixflake.md b/doc/manual/rl-next/9063-introduce-libnixflake.md new file mode 100644 index 000000000..fd3645446 --- /dev/null +++ b/doc/manual/rl-next/9063-introduce-libnixflake.md @@ -0,0 +1,12 @@ +--- +synopsis: "Introduce `libnixflake`" +prs: 9063 +--- + +A new library, `libnixflake`, has been introduced to better separate the Flakes layer within Nix. This change refactors the codebase to encapsulate Flakes-specific functionality within its own library. + +See the commits in the pull request for detailed changes, with the only significant code modifications happening in the initial commit. + +This change was alluded to in [RFC 134](https://github.com/nixos/rfcs/blob/master/rfcs/0134-nix-store-layer.md) and is a step towards a more modular and maintainable codebase. + +Author: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314) From e92dd06a7b5ff00e8908e9a7b5699de56d55e8d6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jul 2024 00:00:52 -0400 Subject: [PATCH 046/142] build-remote: Cope with long store URLs by falling back on hashing I hit this in the Meson port of the functional tests, because the use of standalone build directories. --- src/build-remote/build-remote.cc | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index a0a404e57..8482b742d 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -264,7 +264,20 @@ connected: auto inputs = readStrings(source); auto wantedOutputs = readStrings(source); - AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri.render()) + ".upload-lock", true); + AutoCloseFD uploadLock; + { + auto setUpdateLock = [&](auto && fileName){ + uploadLock = openLockFile(currentLoad + "/" + escapeUri(fileName) + ".upload-lock", true); + }; + try { + setUpdateLock(storeUri.render()); + } catch (SysError & e) { + if (e.errNo != ENAMETOOLONG) throw; + // Try again hashing the store URL so we have a shorter path + auto h = hashString(HashAlgorithm::MD5, storeUri.render()); + setUpdateLock(h.to_string(HashFormat::Base64, false)); + } + } { Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri.render())); From b711fcbef9701ae1fd11839aa911e7737f8a23cd Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 25 Jul 2024 06:00:59 +0200 Subject: [PATCH 047/142] rl-next: Drop zzz-other. Number soup. --- doc/manual/rl-next/zzz-other.md | 50 --------------------------------- 1 file changed, 50 deletions(-) delete mode 100644 doc/manual/rl-next/zzz-other.md diff --git a/doc/manual/rl-next/zzz-other.md b/doc/manual/rl-next/zzz-other.md deleted file mode 100644 index f3721bd38..000000000 --- a/doc/manual/rl-next/zzz-other.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -synopsis: Other changes ---- - -- [#9063](https://github.com/NixOS/nix/pull/9063): introduce `libnixflake` and move flakes-specific code there (Nix is internally composed of a set of libraries, and this better reflects the architecture wrt flakes) -- [#10852](https://github.com/NixOS/nix/pull/10852): make Nix commands respond better to interruption - Ctrl+C in the terminal -- [#10853](https://github.com/NixOS/nix/pull/10853): fix docs of `builtins.importNative` -- [#10858](https://github.com/NixOS/nix/pull/10858): `flake check`: Recognize well known `homeModule`/`homeModules` attribute -- [#10865](https://github.com/NixOS/nix/pull/10865): Update dependencies to Nixpkgs 24.05 (when using the `nix` flake), and support bdwgc 8.2.6 ([#11141](https://github.com/NixOS/nix/issues/11141), [#10880](https://github.com/NixOS/nix/pull/10880)) -- [#10883](https://github.com/NixOS/nix/pull/10883): Use `TMP` instead of `XDG_RUNTIME_DIR` -- [#10994](https://github.com/NixOS/nix/pull/10994): fix minor bug in elided item counts when printing values lazily -- [#10988](https://github.com/NixOS/nix/pull/10988): restore `commit-lockfile-summary` alias -- [#10941](https://github.com/NixOS/nix/pull/10941): invalid derivation name now causes an actionable error message -- Changes to the interaction between Nix's I/O coroutines and the garbage collector: -- [#10878](https://github.com/NixOS/nix/pull/10878): allow `ipc-sysv*` in the Darwin build sandbox -- [#11031](https://github.com/NixOS/nix/pull/11031): fix Darwin build sandbox -- [#10907](https://github.com/NixOS/nix/pull/10907): use opaque struct instead of `void *` in the C API -- [#10947](https://github.com/NixOS/nix/issues/10947): fix evaluation cache accidentally persisting disallowed IFD errors -- [#11020](https://github.com/NixOS/nix/pull/11020): enable fetch and eval caching for tarballs -- [#11041](https://github.com/NixOS/nix/pull/11041): add discovered attribute paths to the `--show-trace` error trace in `nix-build`, `nix-env`, OfBorg, and other callers of `getDerivations` -- [#11056](https://github.com/NixOS/nix/pull/11056): `s3` store now uses system defined proxy settings -- [#11077](https://github.com/NixOS/nix/pull/11077): support hardlinks in tarballs -- [#11100](https://github.com/NixOS/nix/pull/11100): pretty print values consistently regardless of prior thunk state -- [#11086](https://github.com/NixOS/nix/pull/11086): fix loss of evaluation cache additions in `nix env run`, `nix shell`, `nix develop`, and `nix fmt` -- [#11149](https://github.com/NixOS/nix/pull/11149): report GC time and number of GC cycles in `NIX_SHOW_STATS=1` report -- [#11142](https://github.com/NixOS/nix/pull/11142): aliased options can now also be passed as flags, just like their "normal" counterparts, e.g. `--build-max-jobs` now works -- [#11043](https://github.com/NixOS/nix/pull/11043): `assert a == b; e` now reports some detail about why `a` and `b` are different when they are -- [#11159](https://github.com/NixOS/nix/pull/11159): don't crash a nix-daemon worker process when the client disconnects -- Stability improvements and fixes [#10861](https://github.com/NixOS/nix/pull/10861), [#10865](https://github.com/NixOS/nix/pull/10865), [#10918](https://github.com/NixOS/nix/pull/10918), [#10916](https://github.com/NixOS/nix/pull/10916), [#10884](https://github.com/NixOS/nix/pull/10884), [#10943](https://github.com/NixOS/nix/pull/10943), [#11019](https://github.com/NixOS/nix/pull/11019), [#11122](https://github.com/NixOS/nix/pull/11122), [#11117](https://github.com/NixOS/nix/pull/11117) -- User documentation improvements [#10888](https://github.com/NixOS/nix/pull/10888), [#10966](https://github.com/NixOS/nix/pull/10966), [#10974](https://github.com/NixOS/nix/pull/10974), [#10997](https://github.com/NixOS/nix/pull/10997), [#11013](https://github.com/NixOS/nix/pull/11013), [#11059](https://github.com/NixOS/nix/pull/11059), [#11119](https://github.com/NixOS/nix/pull/11119), [#11116](https://github.com/NixOS/nix/pull/11116), [#11061](https://github.com/NixOS/nix/pull/11061), [#11102](https://github.com/NixOS/nix/pull/11102) -- BSD support: [#10896](https://github.com/NixOS/nix/pull/10896) [#11022](https://github.com/NixOS/nix/pull/11022) [#11156](https://github.com/NixOS/nix/pull/11156) -- Windows support: [#10769](https://github.com/NixOS/nix/pull/10769), [#10975](https://github.com/NixOS/nix/pull/10975) [#11153](https://github.com/NixOS/nix/pull/11153) -- Portability: [#7048](https://github.com/NixOS/nix/pull/7048) [#11090](https://github.com/NixOS/nix/pull/11090) -- Installer improvements [#10902](https://github.com/NixOS/nix/pull/10902) -- Performance improvements [#10853](https://github.com/NixOS/nix/pull/10853), [#10854](https://github.com/NixOS/nix/pull/10854), [#11082](https://github.com/NixOS/nix/pull/11082), [#11092](https://github.com/NixOS/nix/pull/11092), [#11113](https://github.com/NixOS/nix/pull/11113) - -Contributor experience improvements: - -Use Meson to build Nix (nearing completion) [#10855](https://github.com/NixOS/nix/pull/10855) [#10904](https://github.com/NixOS/nix/pull/10904) [#10908](https://github.com/NixOS/nix/pull/10908) [#10914](https://github.com/NixOS/nix/pull/10914) [#10933](https://github.com/NixOS/nix/pull/10933) [#10936](https://github.com/NixOS/nix/pull/10936) [#10954](https://github.com/NixOS/nix/pull/10954) [#10955](https://github.com/NixOS/nix/pull/10955) [#10967](https://github.com/NixOS/nix/pull/10967) [#10963](https://github.com/NixOS/nix/pull/10963) [#10973](https://github.com/NixOS/nix/pull/10973) [#11034](https://github.com/NixOS/nix/pull/11034) [#11054](https://github.com/NixOS/nix/pull/11054) [#11055](https://github.com/NixOS/nix/pull/11055) [#11064](https://github.com/NixOS/nix/pull/11064) [#11060](https://github.com/NixOS/nix/pull/11060) [#11155](https://github.com/NixOS/nix/pull/11155) -- Testing improvements [#10864](https://github.com/NixOS/nix/pull/10864), [#10903](https://github.com/NixOS/nix/pull/10903), [#10874](https://github.com/NixOS/nix/pull/10874), [#10922](https://github.com/NixOS/nix/pull/10922), [#11006](https://github.com/NixOS/nix/pull/11006), [#11110](https://github.com/NixOS/nix/pull/11110), [#10931](https://github.com/NixOS/nix/pull/10931), [#11123](https://github.com/NixOS/nix/pull/11123) - - [#10603](https://github.com/NixOS/nix/pull/10603): We now evaluate a set of flakes in CI - - [#10922](https://github.com/NixOS/nix/pull/10922): The functional test suite is now run in both in the build sandbox and in a NixOS environment -- CI improvements [#10929](https://github.com/NixOS/nix/pull/10929) [#10999](https://github.com/NixOS/nix/pull/10999) [#11009](https://github.com/NixOS/nix/pull/11009) [#11065](https://github.com/NixOS/nix/pull/11065) [#11071](https://github.com/NixOS/nix/pull/11071) -- Contributor documentation improvements [#10869](https://github.com/NixOS/nix/pull/10869), [#9871](https://github.com/NixOS/nix/pull/9871), [#10960](https://github.com/NixOS/nix/pull/10960), [#11147](https://github.com/NixOS/nix/pull/11147) -- Error message improvements: [#11050](https://github.com/NixOS/nix/pull/11050) [#11154](https://github.com/NixOS/nix/pull/11154) -- Cleaning up the Settings system (`nix.conf` and related architectural cleanups): [#10913](https://github.com/NixOS/nix/pull/10913), [#10951](https://github.com/NixOS/nix/pull/10951), [#11007](https://github.com/NixOS/nix/pull/11007), [#11108](https://github.com/NixOS/nix/pull/11108), [#11014](https://github.com/NixOS/nix/pull/11014), [#11109](https://github.com/NixOS/nix/pull/11109), [#11112](https://github.com/NixOS/nix/pull/11112) -- Other cleanups and refactors [#10857](https://github.com/NixOS/nix/pull/10857) [#10935](https://github.com/NixOS/nix/pull/10935) [#10873](https://github.com/NixOS/nix/pull/10873) [#10745](https://github.com/NixOS/nix/pull/10745) [#10961](https://github.com/NixOS/nix/pull/10961) [#10962](https://github.com/NixOS/nix/pull/10962) [#10972](https://github.com/NixOS/nix/pull/10972) [#11018](https://github.com/NixOS/nix/pull/11018) [#11035](https://github.com/NixOS/nix/pull/11035) [#11037](https://github.com/NixOS/nix/pull/11037) [#11081](https://github.com/NixOS/nix/pull/11081) [#11089](https://github.com/NixOS/nix/pull/11089) [#11093](https://github.com/NixOS/nix/pull/11093) [#11114](https://github.com/NixOS/nix/pull/11114) [#11103](https://github.com/NixOS/nix/pull/11103) [#11126](https://github.com/NixOS/nix/pull/11126) [#11125](https://github.com/NixOS/nix/pull/11125) [#11120](https://github.com/NixOS/nix/pull/11120) -- Scheduler/builder refactoring [#11005](https://github.com/NixOS/nix/pull/11005) -- [#11011](https://github.com/NixOS/nix/pull/11011): enable `-Werror=unused-result` - From 1ae573831756ac08fd30eb9cfd90692b1c06ed3b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 25 Jul 2024 00:02:43 -0400 Subject: [PATCH 048/142] Fix some warnings I think they came from the last Nixpkgs bump. --- src/libexpr/primops/fromTOML.cc | 4 ++++ src/libstore/s3-binary-cache-store.cc | 3 +++ src/libstore/s3.hh | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 6c7d303e8..70755f9e0 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -2,7 +2,11 @@ #include "eval-inline.hh" #include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wswitch-enum" #include +#pragma GCC diagnostic pop namespace nix { diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 92ab47cd6..1082657bb 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -10,6 +10,8 @@ #include "compression.hh" #include "filetransfer.hh" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wswitch-enum" #include #include #include @@ -25,6 +27,7 @@ #include #include #include +#pragma GCC diagnostic pop using namespace Aws::Transfer; diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh index f0aeb3bed..18de115ae 100644 --- a/src/libstore/s3.hh +++ b/src/libstore/s3.hh @@ -8,7 +8,7 @@ #include #include -namespace Aws { namespace Client { class ClientConfiguration; } } +namespace Aws { namespace Client { struct ClientConfiguration; } } namespace Aws { namespace S3 { class S3Client; } } namespace nix { From 6c38bc09526eab2ad66083e339f0c10cf7ce813a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 25 Jul 2024 07:34:08 +0200 Subject: [PATCH 049/142] {src/perl,build-utils-meson/diagnostics}: sort cflags This makes them easier to copy between places. --- build-utils-meson/diagnostics/meson.build | 6 +++--- src/perl/meson.build | 20 ++++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/build-utils-meson/diagnostics/meson.build b/build-utils-meson/diagnostics/meson.build index 2b79f6566..4548f93ad 100644 --- a/build-utils-meson/diagnostics/meson.build +++ b/build-utils-meson/diagnostics/meson.build @@ -1,11 +1,11 @@ add_project_arguments( - '-Wno-deprecated-declarations', - '-Wimplicit-fallthrough', + '-Wdeprecated-copy', '-Werror=switch', '-Werror=switch-enum', '-Werror=unused-result', - '-Wdeprecated-copy', '-Wignored-qualifiers', + '-Wimplicit-fallthrough', + '-Wno-deprecated-declarations', # Enable assertions in libstdc++ by default. Harmless on libc++. Benchmarked # at ~1% overhead in `nix search`. # diff --git a/src/perl/meson.build b/src/perl/meson.build index 5fe7e1e28..02e0e68e5 100644 --- a/src/perl/meson.build +++ b/src/perl/meson.build @@ -23,21 +23,21 @@ nix_perl_conf.set('PACKAGE_VERSION', meson.project_version()) # set error arguments #------------------------------------------------- error_args = [ - '-Werror=unused-result', '-Wdeprecated-copy', '-Wdeprecated-declarations', + '-Werror=unused-result', '-Wignored-qualifiers', - '-Wno-pedantic', - '-Wno-non-virtual-dtor', - '-Wno-unused-parameter', - '-Wno-variadic-macros', - '-Wno-missing-field-initializers', - '-Wno-unknown-warning-option', - '-Wno-unused-variable', - '-Wno-literal-suffix', - '-Wno-reserved-user-defined-literal', '-Wno-duplicate-decl-specifier', + '-Wno-literal-suffix', + '-Wno-missing-field-initializers', + '-Wno-non-virtual-dtor', + '-Wno-pedantic', '-Wno-pointer-bool-conversion', + '-Wno-reserved-user-defined-literal', + '-Wno-unknown-warning-option', + '-Wno-unused-parameter', + '-Wno-unused-variable', + '-Wno-variadic-macros', ] add_project_arguments( From 2c07ea8abbb2804b476b9df21a9c7c3c2e8dbb11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 25 Jul 2024 07:40:31 +0200 Subject: [PATCH 050/142] build-utils-meson: remove oudated meson build comment --- build-utils-meson/diagnostics/meson.build | 3 --- 1 file changed, 3 deletions(-) diff --git a/build-utils-meson/diagnostics/meson.build b/build-utils-meson/diagnostics/meson.build index 4548f93ad..e81f19eff 100644 --- a/build-utils-meson/diagnostics/meson.build +++ b/build-utils-meson/diagnostics/meson.build @@ -6,8 +6,5 @@ add_project_arguments( '-Wignored-qualifiers', '-Wimplicit-fallthrough', '-Wno-deprecated-declarations', - # Enable assertions in libstdc++ by default. Harmless on libc++. Benchmarked - # at ~1% overhead in `nix search`. - # language : 'cpp', ) From 63e50a4b56a2e21e854636a622941fd36d2e78da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 25 Jul 2024 07:32:03 +0200 Subject: [PATCH 051/142] add werror=suggest-override Improves code readability by making overrides explicit. Inspired by lix code-base --- Makefile | 2 +- build-utils-meson/diagnostics/meson.build | 1 + src/libexpr/json-to-value.cc | 26 +++++++++---------- src/libexpr/nixexpr.hh | 4 +-- src/libstore/daemon.cc | 2 +- .../unix/build/local-derivation-goal.cc | 2 +- src/libutil/serialise.cc | 4 +-- src/libutil/serialise.hh | 6 ++--- src/perl/meson.build | 1 + 9 files changed, 25 insertions(+), 23 deletions(-) diff --git a/Makefile b/Makefile index bb64a104e..ac76b1532 100644 --- a/Makefile +++ b/Makefile @@ -92,7 +92,7 @@ ifdef HOST_WINDOWS GLOBAL_LDFLAGS += -Wl,--export-all-symbols endif -GLOBAL_CXXFLAGS += -g -Wall -Wdeprecated-copy -Wignored-qualifiers -Wimplicit-fallthrough -Werror=unused-result -include $(buildprefix)config.h -std=c++2a -I src +GLOBAL_CXXFLAGS += -g -Wall -Wdeprecated-copy -Wignored-qualifiers -Wimplicit-fallthrough -Werror=unused-result -Werror=suggest-override -include $(buildprefix)config.h -std=c++2a -I src # Include the main lib, causing rules to be defined diff --git a/build-utils-meson/diagnostics/meson.build b/build-utils-meson/diagnostics/meson.build index e81f19eff..30eedfc13 100644 --- a/build-utils-meson/diagnostics/meson.build +++ b/build-utils-meson/diagnostics/meson.build @@ -1,5 +1,6 @@ add_project_arguments( '-Wdeprecated-copy', + '-Werror=suggest-override', '-Werror=switch', '-Werror=switch-enum', '-Werror=unused-result', diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 20bee193f..33ab55ee9 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -80,42 +80,42 @@ class JSONSax : nlohmann::json_sax { public: JSONSax(EvalState & state, Value & v) : state(state), rs(new JSONState(&v)) {}; - bool null() + bool null() override { rs->value(state).mkNull(); rs->add(); return true; } - bool boolean(bool val) + bool boolean(bool val) override { rs->value(state).mkBool(val); rs->add(); return true; } - bool number_integer(number_integer_t val) + bool number_integer(number_integer_t val) override { rs->value(state).mkInt(val); rs->add(); return true; } - bool number_unsigned(number_unsigned_t val) + bool number_unsigned(number_unsigned_t val) override { rs->value(state).mkInt(val); rs->add(); return true; } - bool number_float(number_float_t val, const string_t & s) + bool number_float(number_float_t val, const string_t & s) override { rs->value(state).mkFloat(val); rs->add(); return true; } - bool string(string_t & val) + bool string(string_t & val) override { rs->value(state).mkString(val); rs->add(); @@ -123,7 +123,7 @@ public: } #if NLOHMANN_JSON_VERSION_MAJOR >= 3 && NLOHMANN_JSON_VERSION_MINOR >= 8 - bool binary(binary_t&) + bool binary(binary_t&) override { // This function ought to be unreachable assert(false); @@ -131,35 +131,35 @@ public: } #endif - bool start_object(std::size_t len) + bool start_object(std::size_t len) override { rs = std::make_unique(std::move(rs)); return true; } - bool key(string_t & name) + bool key(string_t & name) override { dynamic_cast(rs.get())->key(name, state); return true; } - bool end_object() { + bool end_object() override { rs = rs->resolve(state); rs->add(); return true; } - bool end_array() { + bool end_array() override { return end_object(); } - bool start_array(size_t len) { + bool start_array(size_t len) override { rs = std::make_unique(std::move(rs), len != std::numeric_limits::max() ? len : 128); return true; } - bool parse_error(std::size_t, const std::string&, const nlohmann::detail::exception& ex) { + bool parse_error(std::size_t, const std::string&, const nlohmann::detail::exception& ex) override { throw JSONParseError("%s", ex.what()); } }; diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 1bcc962c5..3279e3d48 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -186,7 +186,7 @@ struct ExprInheritFrom : ExprVar this->fromWith = nullptr; } - void bindVars(EvalState & es, const std::shared_ptr & env); + void bindVars(EvalState & es, const std::shared_ptr & env) override; }; struct ExprSelect : Expr @@ -203,7 +203,7 @@ struct ExprSelect : Expr * * @param[out] v The attribute set that should contain the last attribute name (if it exists). * @return The last attribute name in `attrPath` - * + * * @note This does *not* evaluate the final attribute, and does not fail if that's the only attribute that does not exist. */ Symbol evalExceptFinalSelect(EvalState & state, Env & env, Value & attrs); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 6533b2f58..f28f92fce 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -167,7 +167,7 @@ struct TunnelSink : Sink { Sink & to; TunnelSink(Sink & to) : to(to) { } - void operator () (std::string_view data) + void operator () (std::string_view data) override { to << STDERR_WRITE; writeString(data, to); diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 0dd102200..d30caaf51 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -1258,7 +1258,7 @@ bool LocalDerivationGoal::isAllowed(const DerivedPath & req) struct RestrictedStoreConfig : virtual LocalFSStoreConfig { using LocalFSStoreConfig::LocalFSStoreConfig; - const std::string name() { return "Restricted Store"; } + const std::string name() override { return "Restricted Store"; } }; /* A wrapper around LocalStore that only allows building/querying of diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 36b99905a..1be7fa37a 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -190,11 +190,11 @@ struct VirtualStackAllocator { class DefaultStackAllocator : public StackAllocator { boost::coroutines2::default_stack stack; - boost::context::stack_context allocate() { + boost::context::stack_context allocate() override { return stack.allocate(); } - void deallocate(boost::context::stack_context sctx) { + void deallocate(boost::context::stack_context sctx) override { stack.deallocate(sctx); } }; diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 8137db5f4..c7290dcef 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -204,7 +204,7 @@ struct TeeSink : Sink { Sink & sink1, & sink2; TeeSink(Sink & sink1, Sink & sink2) : sink1(sink1), sink2(sink2) { } - virtual void operator () (std::string_view data) + virtual void operator () (std::string_view data) override { sink1(data); sink2(data); @@ -221,7 +221,7 @@ struct TeeSource : Source Sink & sink; TeeSource(Source & orig, Sink & sink) : orig(orig), sink(sink) { } - size_t read(char * data, size_t len) + size_t read(char * data, size_t len) override { size_t n = orig.read(data, len); sink({data, n}); @@ -238,7 +238,7 @@ struct SizedSource : Source size_t remain; SizedSource(Source & orig, size_t size) : orig(orig), remain(size) { } - size_t read(char * data, size_t len) + size_t read(char * data, size_t len) override { if (this->remain <= 0) { throw EndOfFile("sized: unexpected end-of-file"); diff --git a/src/perl/meson.build b/src/perl/meson.build index 02e0e68e5..dcb6a68a4 100644 --- a/src/perl/meson.build +++ b/src/perl/meson.build @@ -25,6 +25,7 @@ nix_perl_conf.set('PACKAGE_VERSION', meson.project_version()) error_args = [ '-Wdeprecated-copy', '-Wdeprecated-declarations', + '-Werror=suggest-override', '-Werror=unused-result', '-Wignored-qualifiers', '-Wno-duplicate-decl-specifier', From 9b5ce9acc29a98bedf0530d5611f25f57eae6197 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Thu, 25 Jul 2024 09:21:31 +0200 Subject: [PATCH 052/142] build-remote: only allocate storeUri once also it's probably not much overhead compared to the networking stuff it, but it's less code at least. --- src/build-remote/build-remote.cc | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 8482b742d..82ad7d862 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -101,7 +101,7 @@ static int main_build_remote(int argc, char * * argv) } std::optional drvPath; - StoreReference storeUri; + std::string storeUri; while (true) { @@ -234,17 +234,16 @@ static int main_build_remote(int argc, char * * argv) lock = -1; try { + storeUri = bestMachine->storeUri.render(); - Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri.render())); + Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", storeUri)); sshStore = bestMachine->openStore(); sshStore->connect(); - storeUri = bestMachine->storeUri; - } catch (std::exception & e) { auto msg = chomp(drainFD(5, false)); printError("cannot build on '%s': %s%s", - bestMachine->storeUri.render(), e.what(), + storeUri, e.what(), msg.empty() ? "" : ": " + msg); bestMachine->enabled = false; continue; @@ -259,7 +258,7 @@ connected: assert(sshStore); - std::cerr << "# accept\n" << storeUri.render() << "\n"; + std::cerr << "# accept\n" << storeUri << "\n"; auto inputs = readStrings(source); auto wantedOutputs = readStrings(source); @@ -270,17 +269,17 @@ connected: uploadLock = openLockFile(currentLoad + "/" + escapeUri(fileName) + ".upload-lock", true); }; try { - setUpdateLock(storeUri.render()); + setUpdateLock(storeUri); } catch (SysError & e) { if (e.errNo != ENAMETOOLONG) throw; // Try again hashing the store URL so we have a shorter path - auto h = hashString(HashAlgorithm::MD5, storeUri.render()); + auto h = hashString(HashAlgorithm::MD5, storeUri); setUpdateLock(h.to_string(HashFormat::Base64, false)); } } { - Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri.render())); + Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri)); auto old = signal(SIGALRM, handleAlarm); alarm(15 * 60); @@ -293,7 +292,7 @@ connected: auto substitute = settings.buildersUseSubstitutes ? Substitute : NoSubstitute; { - Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri.render())); + Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri)); copyPaths(*store, *sshStore, store->parseStorePathSet(inputs), NoRepair, NoCheckSigs, substitute); } @@ -331,7 +330,7 @@ connected: optResult = sshStore->buildDerivation(*drvPath, (const BasicDerivation &) drv); auto & result = *optResult; if (!result.success()) - throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri.render(), result.errorMsg); + throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg); } else { copyClosure(*store, *sshStore, StorePathSet {*drvPath}, NoRepair, NoCheckSigs, substitute); auto res = sshStore->buildPathsWithResults({ @@ -374,7 +373,7 @@ connected: } if (!missingPaths.empty()) { - Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri.render())); + Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri)); if (auto localStore = store.dynamic_pointer_cast()) for (auto & path : missingPaths) localStore->locksHeld.insert(store->printStorePath(path)); /* FIXME: ugly */ From baa28159d316e2c506fee8a82f66726c9305f68a Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 25 Jul 2024 15:38:02 +0200 Subject: [PATCH 053/142] Update tests/functional/test-infra.sh Co-authored-by: John Ericson --- tests/functional/test-infra.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/test-infra.sh b/tests/functional/test-infra.sh index 1dab069fb..d02a11b46 100755 --- a/tests/functional/test-infra.sh +++ b/tests/functional/test-infra.sh @@ -111,7 +111,7 @@ unset res # `grepQuiet` does not allow newlines in its arguments, because grep quietly # treats them as multiple queries. -( echo foo; echo bar; ) | expectStderr -101 grepQuiet $'foo\nbar' \ +{ echo foo; echo bar; } | expectStderr -101 grepQuiet $'foo\nbar' \ | grepQuiet -E 'test-infra\.sh:[0-9]+: in call to grepQuiet: newline not allowed in arguments; grep would try each line individually as if connected by an OR operator' # We took the blue pill and woke up in a world where `grep` is moderately safe. From f0fe1d880ded3b5c1e6d44ad0cee9105a50ebd65 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 25 Jul 2024 15:39:15 +0200 Subject: [PATCH 054/142] Update doc/manual/rl-next/10734-nix3-build-show-all-fod-errors-with-keep-going.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jörg Thalheim --- .../10734-nix3-build-show-all-fod-errors-with-keep-going.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/rl-next/10734-nix3-build-show-all-fod-errors-with-keep-going.md b/doc/manual/rl-next/10734-nix3-build-show-all-fod-errors-with-keep-going.md index 5c2797be8..e4e2f797c 100644 --- a/doc/manual/rl-next/10734-nix3-build-show-all-fod-errors-with-keep-going.md +++ b/doc/manual/rl-next/10734-nix3-build-show-all-fod-errors-with-keep-going.md @@ -5,6 +5,6 @@ prs: 10734 The [`nix build`](@docroot@/command-ref/new-cli/nix3-build.md) command has been updated to improve the behavior of the [`--keep-going`] flag. Now, when `--keep-going` is used, all hash-mismatch errors of failing fixed-output derivations (FODs) are displayed, similar to the behavior of `nix build`. This enhancement ensures that all relevant build errors are shown, making it easier for users to update multiple derivations at once or to diagnose and fix issues. -Author: [**Jörg Thalheim (@Mic92)**](https://github.com/Mic92) +Author: [**Jörg Thalheim (@Mic92)**](https://github.com/Mic92), [**Maximilian Bosch (@Ma27)**](https://github.com/Ma27) [`--keep-going`](@docroot@/command-ref/opt-common.md#opt-keep-going) From 55a654abfd0f9a0d8e70b9d2dec410888a3e76db Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Wed, 24 Jul 2024 17:53:17 +0200 Subject: [PATCH 055/142] Make panic() and unreachable() robust Plus one or two tweaks. --- src/libutil/error.cc | 28 +++++++++++++++++++++++++--- src/libutil/error.hh | 8 ++------ 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index b1858911a..ccd008c7c 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -1,3 +1,5 @@ +#include + #include "error.hh" #include "environment-variables.hh" #include "signals.hh" @@ -430,16 +432,36 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s return out; } +/** Write to stderr in a robust and minimal way, considering that the process + * may be in a bad state. + */ +static void writeErr(std::string_view buf) +{ + while (!buf.empty()) { + auto n = write(STDERR_FILENO, buf.data(), buf.size()); + if (n < 0) { + if (errno == EINTR) continue; + abort(); + } + buf = buf.substr(n); + } +} + void panic(std::string_view msg) { - printError(msg); - printError("This was a fatal error, aborting."); + writeErr("\n\n" ANSI_RED "terminating due to unexpected unrecoverable internal error: " ANSI_NORMAL ); + writeErr(msg); + writeErr("\n"); abort(); } void panic(const char * file, int line, const char * func) { - panic(std::string("Unexpected condition in ") + func + " at " + file + ":" + std::to_string(line)); + char buf[512]; + int n = snprintf(buf, sizeof(buf), "Unexpected condition in %s at %s:%d", func, file, line); + if (n < 0) + panic("Unexpected condition and could not format error message"); + panic(std::string_view(buf, std::min(static_cast(sizeof(buf)), n))); } } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 572a1baf7..58d902622 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -275,17 +275,13 @@ void throwExceptionSelfCheck(); /** * Print a message and abort(). - * - * @note: This assumes that the logger is operational */ [[noreturn]] void panic(std::string_view msg); /** * Print a basic error message with source position and abort(). - * Use the unreachable macro to call this. - * - * @note: This assumes that the logger is operational + * Use the unreachable() macro to call this. */ [[noreturn]] void panic(const char * file, int line, const char * func); @@ -295,6 +291,6 @@ void panic(const char * file, int line, const char * func); * * @note: This assumes that the logger is operational */ -#define unreachable() (panic(__FILE__, __LINE__, __func__)) +#define unreachable() (::nix::panic(__FILE__, __LINE__, __func__)) } From 6e178cd899dc9e25e8da6a018f7cfdb233b762ea Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Thu, 25 Jul 2024 11:38:45 -0700 Subject: [PATCH 056/142] Fix reference to experimental features docs Arose because https://github.com/NixOS/nix/pull/9014 merged before https://github.com/NixOS/nix/pull/11131, but the latter did not rebase / merge against the latest master. --- doc/manual/src/language/operators.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index e96a28988..e1c020781 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -198,11 +198,11 @@ Equivalent to `!`*b1* `||` *b2*. > **Warning** > > This syntax is part of an -> [experimental feature](@docroot@/contributing/experimental-features.md) +> [experimental feature](@docroot@/development/experimental-features.md) > and may change in future releases. > > To use this syntax, make sure the -> [`pipe-operators` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-pipe-operators) +> [`pipe-operators` experimental feature](@docroot@/development/experimental-features.md#xp-feature-pipe-operators) > is enabled. > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): > From 90459e60dcfedae27bd7da5615fe1aac25cc4c4e Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Thu, 25 Jul 2024 11:38:45 -0700 Subject: [PATCH 057/142] Fix reference to experimental features docs Arose because https://github.com/NixOS/nix/pull/9014 merged before https://github.com/NixOS/nix/pull/11131, but the latter did not rebase / merge against the latest master. --- doc/manual/src/language/operators.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index e96a28988..e1c020781 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -198,11 +198,11 @@ Equivalent to `!`*b1* `||` *b2*. > **Warning** > > This syntax is part of an -> [experimental feature](@docroot@/contributing/experimental-features.md) +> [experimental feature](@docroot@/development/experimental-features.md) > and may change in future releases. > > To use this syntax, make sure the -> [`pipe-operators` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-pipe-operators) +> [`pipe-operators` experimental feature](@docroot@/development/experimental-features.md#xp-feature-pipe-operators) > is enabled. > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): > From 492715c0bb589f0df963cb9902a9f46230104d44 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Tue, 18 Jun 2024 00:54:05 -0700 Subject: [PATCH 058/142] diff-closures: fix a use after free Found by looking for interesting asan reports from the test suite. What happened here is that name got overwritten, but it was what actually held the backing memory for the thing it got overwritten by, which was a by-reference value coming out of std::regex. Due to absurd reasons I cannot seem to use a string_view iterator here, so I just copy the string with a longer lifetime instead. idk lol ==3796364==ERROR: AddressSanitizer: heap-use-after-free on address 0x503000014c61 at pc 0x74843523bf1d bp 0x7ffc68351330 sp 0x7ffc68350af0 READ of size 3 at 0x503000014c61 thread T0 0 0x74843523bf1c in __asan_memcpy (/nix/store/mzhqknx2mc94jdz4n320hn1lml86398y-clang-wrapper-17.0.6/resource-root/lib/linux/libclang_rt.asan-x86_64.so+0x159f1c) 1 0x6403cf6cbff4 in std::char_traits::copy(char*, char const*, unsigned long) /nix/store/14c6s4xzhy14i2b05s00rjns2j93gzz4-gcc-13.2.0/include/c++/13.2.0/bits/char_traits.h:445:33 <...> 7 0x6403cf6cbff4 in std::__cxx11::sub_match<__gnu_cxx::__normal_iterator, std::allocator>>>::str() const /nix/store/14c6s4xzhy14i2b05s00rjns2j93gzz4-gcc-13.2.0/include/c++/13.2.0/bits/regex.h:966:6 8 0x6403cf6cbff4 in std::__cxx11::sub_match<__gnu_cxx::__normal_iterator, std::allocator>>>::operator std::__cxx11::basic_string, std::allocator>() const /nix/store/14c6s4xzhy14i2b05s00rjns2j93gzz4-gcc-13.2.0/include/c++/13.2.0/bits/regex.h:955:16 9 0x6403cf6cbff4 in nix::getClosureInfo[abi:cxx11](nix::ref, nix::StorePath const&) /home/jade/lix/lix2/build/src/nix/diff-closures.cc:37:26 10 0x6403cf6cd70c in nix::printClosureDiff(nix::ref, nix::StorePath const&, nix::StorePath const&, std::basic_string_view>) /home/jade/lix/lix2/build/src/nix/diff-closures.cc:54:25 11 0x6403cf873331 in CmdProfileDiffClosures::run(nix::ref) /home/jade/lix/lix2/build/src/nix/profile.cc:479:17 <...> 0x503000014c61 is located 17 bytes inside of 21-byte region [0x503000014c50,0x503000014c65) freed by thread T0 here: 0 0x748435250470 in operator delete(void*) (/nix/store/mzhqknx2mc94jdz4n320hn1lml86398y-clang-wrapper-17.0.6/resource-root/lib/linux/libclang_rt.asan-x86_64.so+0x16e470) <...> 6 0x6403cf6cbda2 in std::__cxx11::basic_string, std::allocator>::~basic_string() /nix/store/14c6s4xzhy14i2b05s00rjns2j93gzz4-gcc-13.2.0/include/c++/13.2.0/bits/basic_string.h:792:9 7 0x6403cf6cbda2 in nix::getClosureInfo[abi:cxx11](nix::ref, nix::StorePath const&) /home/jade/lix/lix2/build/src/nix/diff-closures.cc:36:13 8 0x6403cf6cd70c in nix::printClosureDiff(nix::ref, nix::StorePath const&, nix::StorePath const&, std::basic_string_view>) /home/jade/lix/lix2/build/src/nix/diff-closures.cc:54:25 <...> previously allocated by thread T0 here: 0 0x74843524fa38 in operator new(unsigned long) (/nix/store/mzhqknx2mc94jdz4n320hn1lml86398y-clang-wrapper-17.0.6/resource-root/lib/linux/libclang_rt.asan-x86_64.so+0x16da38) <...> 9 0x6403cf6cb68c in std::__cxx11::basic_string, std::allocator>::basic_string>, void>(std::basic_string_view> const&, std::allocator const&) /nix/store/14c6s4xzhy14i2b05s00rjns2j93gzz4-gcc-13.2.0/include/c++/13.2.0/bits/basic_string.h:784:4 10 0x6403cf6cb68c in nix::getClosureInfo[abi:cxx11](nix::ref, nix::StorePath const&) /home/jade/lix/lix2/build/src/nix/diff-closures.cc:33:21 11 0x6403cf6cd70c in nix::printClosureDiff(nix::ref, nix::StorePath const&, nix::StorePath const&, std::basic_string_view>) /home/jade/lix/lix2/build/src/nix/diff-closures.cc:54:25 12 0x6403cf873331 in CmdProfileDiffClosures::run(nix::ref) /home/jade/lix/lix2/build/src/nix/profile.cc:479:17 <...> (cherry-picked from https://git.lix.systems/lix-project/lix/commit/b9b1bbd22fc3b34a4d8e2090f0d033f742c32ee0) --- src/nix/diff-closures.cc | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index 46c94b211..4e12dc60a 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -31,9 +31,18 @@ GroupedPaths getClosureInfo(ref store, const StorePath & toplevel) version suffixes like "unstable"). */ static std::regex regex("(.*)-([a-z]+|lib32|lib64)"); std::smatch match; - std::string name(path.name()); + std::string name{path.name()}; + // Used to keep name alive through being potentially overwritten below + // (to not invalidate the references from the regex result) + // + // n.b. cannot be just path.name().{begin,end}() since that returns const + // char *, which does not, for some reason, convert as required on + // libstdc++. Seems like a libstdc++ bug or standard bug to me... we + // can afford the allocation in any case. + const std::string origName{path.name()}; std::string outputName; - if (std::regex_match(name, match, regex)) { + + if (std::regex_match(origName, match, regex)) { name = match[1]; outputName = match[2]; } From 07aeedd37e2e6c2b2d48fd5cf59917adc241f949 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Tue, 23 Jul 2024 21:45:30 +0200 Subject: [PATCH 059/142] diff-closures: remove gratuitous copy This was done originally because std::smatch does not accept `const char *` as iterators. However, this was because we should have been using std::cmatch instead. (cherry picked from commit https://git.lix.systems/lix-project/lix/commit/12a5838d11f790d36e2ac626e8916a2c19bcb80b) --- src/nix/diff-closures.cc | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index 4e12dc60a..2bc7fe82b 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -25,24 +25,17 @@ GroupedPaths getClosureInfo(ref store, const StorePath & toplevel) GroupedPaths groupedPaths; - for (auto & path : closure) { + for (auto const & path : closure) { /* Strip the output name. Unfortunately this is ambiguous (we can't distinguish between output names like "bin" and version suffixes like "unstable"). */ static std::regex regex("(.*)-([a-z]+|lib32|lib64)"); - std::smatch match; + std::cmatch match; std::string name{path.name()}; - // Used to keep name alive through being potentially overwritten below - // (to not invalidate the references from the regex result) - // - // n.b. cannot be just path.name().{begin,end}() since that returns const - // char *, which does not, for some reason, convert as required on - // libstdc++. Seems like a libstdc++ bug or standard bug to me... we - // can afford the allocation in any case. - const std::string origName{path.name()}; + std::string_view const origName = path.name(); std::string outputName; - if (std::regex_match(origName, match, regex)) { + if (std::regex_match(origName.begin(), origName.end(), match, regex)) { name = match[1]; outputName = match[2]; } From 429a197d246782e286fe7d9f6fad0b9daf2941ec Mon Sep 17 00:00:00 2001 From: Ryan Hendrickson Date: Sat, 20 Jul 2024 12:04:25 -0400 Subject: [PATCH 060/142] parser.y: use names where I'll be refactoring --- src/libexpr/parser.y | 72 ++++++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 9ad41c148..256244a75 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -180,22 +180,22 @@ expr_function $$ = me; SET_DOC_POS(me, @1); } - | '{' formals '}' ':' expr_function - { auto me = new ExprLambda(CUR_POS, state->validateFormals($2), $5); + | '{' formals '}' ':' expr_function[body] + { auto me = new ExprLambda(CUR_POS, state->validateFormals($formals), $body); $$ = me; SET_DOC_POS(me, @1); } - | '{' formals '}' '@' ID ':' expr_function + | '{' formals '}' '@' ID ':' expr_function[body] { - auto arg = state->symbols.create($5); - auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($2, CUR_POS, arg), $7); + auto arg = state->symbols.create($ID); + auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($formals, CUR_POS, arg), $body); $$ = me; SET_DOC_POS(me, @1); } - | ID '@' '{' formals '}' ':' expr_function + | ID '@' '{' formals '}' ':' expr_function[body] { - auto arg = state->symbols.create($1); - auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($4, CUR_POS, arg), $7); + auto arg = state->symbols.create($ID); + auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($formals, CUR_POS, arg), $body); $$ = me; SET_DOC_POS(me, @1); } @@ -364,50 +364,50 @@ ind_string_parts ; binds - : binds attrpath '=' expr ';' { - $$ = $1; + : binds[accum] attrpath '=' expr ';' { + $$ = $accum; - auto pos = state->at(@2); - auto exprPos = state->at(@4); + auto pos = state->at(@attrpath); + auto exprPos = state->at(@expr); { auto it = state->lexerState.positionToDocComment.find(pos); if (it != state->lexerState.positionToDocComment.end()) { - $4->setDocComment(it->second); + $expr->setDocComment(it->second); state->lexerState.positionToDocComment.emplace(exprPos, it->second); } } - state->addAttr($$, std::move(*$2), $4, pos); - delete $2; + state->addAttr($$, std::move(*$attrpath), $expr, pos); + delete $attrpath; } - | binds INHERIT attrs ';' - { $$ = $1; - for (auto & [i, iPos] : *$3) { - if ($$->attrs.find(i.symbol) != $$->attrs.end()) - state->dupAttr(i.symbol, iPos, $$->attrs[i.symbol].pos); - $$->attrs.emplace( + | binds[accum] INHERIT attrs ';' + { $$ = $accum; + for (auto & [i, iPos] : *$attrs) { + if ($accum->attrs.find(i.symbol) != $accum->attrs.end()) + state->dupAttr(i.symbol, iPos, $accum->attrs[i.symbol].pos); + $accum->attrs.emplace( i.symbol, ExprAttrs::AttrDef(new ExprVar(iPos, i.symbol), iPos, ExprAttrs::AttrDef::Kind::Inherited)); } - delete $3; + delete $attrs; } - | binds INHERIT '(' expr ')' attrs ';' - { $$ = $1; - if (!$$->inheritFromExprs) - $$->inheritFromExprs = std::make_unique>(); - $$->inheritFromExprs->push_back($4); - auto from = new nix::ExprInheritFrom(state->at(@4), $$->inheritFromExprs->size() - 1); - for (auto & [i, iPos] : *$6) { - if ($$->attrs.find(i.symbol) != $$->attrs.end()) - state->dupAttr(i.symbol, iPos, $$->attrs[i.symbol].pos); - $$->attrs.emplace( + | binds[accum] INHERIT '(' expr ')' attrs ';' + { $$ = $accum; + if (!$accum->inheritFromExprs) + $accum->inheritFromExprs = std::make_unique>(); + $accum->inheritFromExprs->push_back($expr); + auto from = new nix::ExprInheritFrom(state->at(@expr), $accum->inheritFromExprs->size() - 1); + for (auto & [i, iPos] : *$attrs) { + if ($accum->attrs.find(i.symbol) != $accum->attrs.end()) + state->dupAttr(i.symbol, iPos, $accum->attrs[i.symbol].pos); + $accum->attrs.emplace( i.symbol, ExprAttrs::AttrDef( new ExprSelect(iPos, from, i.symbol), iPos, ExprAttrs::AttrDef::Kind::InheritedFrom)); } - delete $6; + delete $attrs; } | { $$ = new ExprAttrs(state->at(@0)); } ; @@ -468,10 +468,10 @@ expr_list ; formals - : formal ',' formals - { $$ = $3; $$->formals.emplace_back(*$1); delete $1; } + : formal ',' formals[accum] + { $$ = $accum; $$->formals.emplace_back(*$formal); delete $formal; } | formal - { $$ = new Formals; $$->formals.emplace_back(*$1); $$->ellipsis = false; delete $1; } + { $$ = new Formals; $$->formals.emplace_back(*$formal); $$->ellipsis = false; delete $formal; } | { $$ = new Formals; $$->ellipsis = false; } | ELLIPSIS From b0a8430e851790ebac4e2324084bd6d86b995316 Mon Sep 17 00:00:00 2001 From: Ryan Hendrickson Date: Sat, 20 Jul 2024 12:04:25 -0400 Subject: [PATCH 061/142] parser.y: move attr doc setting into addAttr --- src/libexpr/parser-state.hh | 11 +++++++++-- src/libexpr/parser.y | 13 +------------ 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/src/libexpr/parser-state.hh b/src/libexpr/parser-state.hh index 4bb5c9204..c23ef32a5 100644 --- a/src/libexpr/parser-state.hh +++ b/src/libexpr/parser-state.hh @@ -86,7 +86,7 @@ struct ParserState void dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos); void dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos); - void addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * e, const PosIdx pos); + void addAttr(ExprAttrs * attrs, AttrPath && attrPath, const ParserLocation & loc, Expr * e, const ParserLocation & exprLoc); Formals * validateFormals(Formals * formals, PosIdx pos = noPos, Symbol arg = {}); Expr * stripIndentation(const PosIdx pos, std::vector>> && es); @@ -110,11 +110,12 @@ inline void ParserState::dupAttr(Symbol attr, const PosIdx pos, const PosIdx pre }); } -inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * e, const PosIdx pos) +inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, const ParserLocation & loc, Expr * e, const ParserLocation & exprLoc) { AttrPath::iterator i; // All attrpaths have at least one attr assert(!attrPath.empty()); + auto pos = at(loc); // Checking attrPath validity. // =========================== for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { @@ -179,6 +180,12 @@ inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * } else { attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos)); } + + auto it = lexerState.positionToDocComment.find(pos); + if (it != lexerState.positionToDocComment.end()) { + e->setDocComment(it->second); + lexerState.positionToDocComment.emplace(at(exprLoc), it->second); + } } inline Formals * ParserState::validateFormals(Formals * formals, PosIdx pos, Symbol arg) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 256244a75..6386747f5 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -366,18 +366,7 @@ ind_string_parts binds : binds[accum] attrpath '=' expr ';' { $$ = $accum; - - auto pos = state->at(@attrpath); - auto exprPos = state->at(@expr); - { - auto it = state->lexerState.positionToDocComment.find(pos); - if (it != state->lexerState.positionToDocComment.end()) { - $expr->setDocComment(it->second); - state->lexerState.positionToDocComment.emplace(exprPos, it->second); - } - } - - state->addAttr($$, std::move(*$attrpath), $expr, pos); + state->addAttr($$, std::move(*$attrpath), @attrpath, $expr, @expr); delete $attrpath; } | binds[accum] INHERIT attrs ';' From 6e3b9e6a4de7430c8b130a7c87d7f5df68cf6c86 Mon Sep 17 00:00:00 2001 From: Ryan Hendrickson Date: Sat, 20 Jul 2024 12:04:25 -0400 Subject: [PATCH 062/142] parser.y: eliminate conflicts --- src/libexpr/parser.y | 61 ++++++++++++------- .../lang/parse-fail-undef-var-2.err.exp | 2 +- 2 files changed, 39 insertions(+), 24 deletions(-) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 6386747f5..2069931e1 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -8,8 +8,8 @@ %parse-param { nix::ParserState * state } %lex-param { void * scanner } %lex-param { nix::ParserState * state } -%expect 1 -%expect-rr 1 +%expect 0 +%expect-rr 0 %code requires { @@ -133,8 +133,8 @@ static Expr * makeCall(PosIdx pos, Expr * fn, Expr * arg) { %type expr_select expr_simple expr_app %type expr_pipe_from expr_pipe_into %type expr_list -%type binds -%type formals +%type binds binds1 +%type formals formal_set %type formal %type attrpath %type attrs @@ -180,22 +180,22 @@ expr_function $$ = me; SET_DOC_POS(me, @1); } - | '{' formals '}' ':' expr_function[body] - { auto me = new ExprLambda(CUR_POS, state->validateFormals($formals), $body); + | formal_set ':' expr_function[body] + { auto me = new ExprLambda(CUR_POS, state->validateFormals($formal_set), $body); $$ = me; SET_DOC_POS(me, @1); } - | '{' formals '}' '@' ID ':' expr_function[body] + | formal_set '@' ID ':' expr_function[body] { auto arg = state->symbols.create($ID); - auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($formals, CUR_POS, arg), $body); + auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($formal_set, CUR_POS, arg), $body); $$ = me; SET_DOC_POS(me, @1); } - | ID '@' '{' formals '}' ':' expr_function[body] + | ID '@' formal_set ':' expr_function[body] { auto arg = state->symbols.create($ID); - auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($formals, CUR_POS, arg), $body); + auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($formal_set, CUR_POS, arg), $body); $$ = me; SET_DOC_POS(me, @1); } @@ -311,11 +311,13 @@ expr_simple /* Let expressions `let {..., body = ...}' are just desugared into `(rec {..., body = ...}).body'. */ | LET '{' binds '}' - { $3->recursive = true; $$ = new ExprSelect(noPos, $3, state->s.body); } + { $3->recursive = true; $3->pos = CUR_POS; $$ = new ExprSelect(noPos, $3, state->s.body); } | REC '{' binds '}' - { $3->recursive = true; $$ = $3; } - | '{' binds '}' - { $$ = $2; } + { $3->recursive = true; $3->pos = CUR_POS; $$ = $3; } + | '{' binds1 '}' + { $2->pos = CUR_POS; $$ = $2; } + | '{' '}' + { $$ = new ExprAttrs(CUR_POS); } | '[' expr_list ']' { $$ = $2; } ; @@ -364,8 +366,13 @@ ind_string_parts ; binds - : binds[accum] attrpath '=' expr ';' { - $$ = $accum; + : binds1 + | { $$ = new ExprAttrs; } + ; + +binds1 + : binds1[accum] attrpath '=' expr ';' + { $$ = $accum; state->addAttr($$, std::move(*$attrpath), @attrpath, $expr, @expr); delete $attrpath; } @@ -398,7 +405,11 @@ binds } delete $attrs; } - | { $$ = new ExprAttrs(state->at(@0)); } + | attrpath '=' expr ';' + { $$ = new ExprAttrs; + state->addAttr($$, std::move(*$attrpath), @attrpath, $expr, @expr); + delete $attrpath; + } ; attrs @@ -456,15 +467,19 @@ expr_list | { $$ = new ExprList; } ; +formal_set + : '{' formals ',' ELLIPSIS '}' { $$ = $formals; $$->ellipsis = true; } + | '{' ELLIPSIS '}' { $$ = new Formals; $$->ellipsis = true; } + | '{' formals ',' '}' { $$ = $formals; $$->ellipsis = false; } + | '{' formals '}' { $$ = $formals; $$->ellipsis = false; } + | '{' '}' { $$ = new Formals; $$->ellipsis = false; } + ; + formals - : formal ',' formals[accum] + : formals[accum] ',' formal { $$ = $accum; $$->formals.emplace_back(*$formal); delete $formal; } | formal - { $$ = new Formals; $$->formals.emplace_back(*$formal); $$->ellipsis = false; delete $formal; } - | - { $$ = new Formals; $$->ellipsis = false; } - | ELLIPSIS - { $$ = new Formals; $$->ellipsis = true; } + { $$ = new Formals; $$->formals.emplace_back(*$formal); delete $formal; } ; formal diff --git a/tests/functional/lang/parse-fail-undef-var-2.err.exp b/tests/functional/lang/parse-fail-undef-var-2.err.exp index 393c454dd..96e87b2aa 100644 --- a/tests/functional/lang/parse-fail-undef-var-2.err.exp +++ b/tests/functional/lang/parse-fail-undef-var-2.err.exp @@ -1,4 +1,4 @@ -error: syntax error, unexpected ':', expecting '}' +error: syntax error, unexpected ':', expecting '}' or ',' at «stdin»:3:13: 2| 3| f = {x, y : ["baz" "bar" z "bat"]}: x + y; From 18db46a6cb72acaae748833b09b428a7794bd9c8 Mon Sep 17 00:00:00 2001 From: Ryan Hendrickson Date: Mon, 22 Jul 2024 11:36:09 -0400 Subject: [PATCH 063/142] parser.y: GLR -> LALR --- src/libexpr/parser-state.hh | 1 + src/libexpr/parser.y | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/libexpr/parser-state.hh b/src/libexpr/parser-state.hh index c23ef32a5..8ad0d9ad7 100644 --- a/src/libexpr/parser-state.hh +++ b/src/libexpr/parser-state.hh @@ -20,6 +20,7 @@ struct StringToken operator std::string_view() const { return {p, l}; } }; +// This type must be trivially copyable; see YYLTYPE_IS_TRIVIAL in parser.y. struct ParserLocation { int beginOffset; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 2069931e1..f2ccca7fc 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -1,4 +1,4 @@ -%glr-parser +%define api.location.type { ::nix::ParserLocation } %define api.pure %locations %define parse.error verbose @@ -9,7 +9,6 @@ %lex-param { void * scanner } %lex-param { nix::ParserState * state } %expect 0 -%expect-rr 0 %code requires { @@ -27,7 +26,17 @@ #include "eval-settings.hh" #include "parser-state.hh" -#define YYLTYPE ::nix::ParserLocation +// Bison seems to have difficulty growing the parser stack when using C++ with +// a custom location type. This undocumented macro tells Bison that our +// location type is "trivially copyable" in C++-ese, so it is safe to use the +// same memcpy macro it uses to grow the stack that it uses with its own +// default location type. Without this, we get "error: memory exhausted" when +// parsing some large Nix files. Our other options are to increase the initial +// stack size (200 by default) to be as large as we ever want to support (so +// that growing the stack is unnecessary), or redefine the stack-relocation +// macro ourselves (which is also undocumented). +#define YYLTYPE_IS_TRIVIAL 1 + #define YY_DECL int yylex \ (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParserState * state) @@ -77,7 +86,7 @@ YY_DECL; using namespace nix; -#define CUR_POS state->at(*yylocp) +#define CUR_POS state->at(yyloc) void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * error) From 2141a52ca3e150079bce8bc12a0a992b16c046f1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Jul 2024 15:38:44 +0200 Subject: [PATCH 064/142] nix repl: Remove unnecessary call to evalString This crashes with the multithreaded evaluator, which checks against attempts to finish an already finished value. --- src/libcmd/repl.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index b5d0816dd..f5e836f8c 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -644,9 +644,6 @@ ProcessLineResult NixRepl::processLine(std::string line) fallbackPos = attr->pos; fallbackDoc = state->getDocCommentForPos(fallbackPos); } - - } else { - evalString(arg, v); } evalString(arg, v); From 6d843ce9fec37e854e518db19ce880895f7de2ac Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 6 Jun 2024 19:32:46 +0200 Subject: [PATCH 065/142] Provide std::hash --- src/libexpr/json-to-value.cc | 2 +- src/libexpr/symbol-table.hh | 11 +++++++++++ src/libexpr/value.hh | 4 ++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 33ab55ee9..21074bdd8 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -42,7 +42,7 @@ class JSONSax : nlohmann::json_sax { auto attrs2 = state.buildBindings(attrs.size()); for (auto & i : attrs) attrs2.insert(i.first, i.second); - parent->value(state).mkAttrs(attrs2.alreadySorted()); + parent->value(state).mkAttrs(attrs2); return std::move(parent); } void add() override { v = nullptr; } diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh index c7a3563b0..8f7257e01 100644 --- a/src/libexpr/symbol-table.hh +++ b/src/libexpr/symbol-table.hh @@ -69,6 +69,8 @@ public: auto operator<=>(const Symbol other) const { return id <=> other.id; } bool operator==(const Symbol other) const { return id == other.id; } + + friend class std::hash; }; /** @@ -132,3 +134,12 @@ public: }; } + +template<> +struct std::hash +{ + std::size_t operator()(const nix::Symbol & s) const noexcept + { + return std::hash{}(s.id); + } +}; diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 1f4d72d39..bfe526db3 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -494,11 +494,11 @@ void Value::mkBlackhole() #if HAVE_BOEHMGC typedef std::vector> ValueVector; -typedef std::map, traceable_allocator>> ValueMap; +typedef std::unordered_map, std::equal_to, traceable_allocator>> ValueMap; typedef std::map, traceable_allocator>> ValueVectorMap; #else typedef std::vector ValueVector; -typedef std::map ValueMap; +typedef std::unordered_map ValueMap; typedef std::map ValueVectorMap; #endif From ce663d75e36f1bb41c11a9d666d5b649328bee92 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 4 Jan 2024 15:05:20 +0100 Subject: [PATCH 066/142] LRUCache: Mark size() as const --- src/libutil/lru-cache.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/lru-cache.hh b/src/libutil/lru-cache.hh index 0e19517ed..6e14cac35 100644 --- a/src/libutil/lru-cache.hh +++ b/src/libutil/lru-cache.hh @@ -89,7 +89,7 @@ public: return i->second.second; } - size_t size() + size_t size() const { return data.size(); } From ea46264bd35d479ca60b3f725d31f0735863281d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Jul 2024 16:14:03 +0200 Subject: [PATCH 067/142] Store: Use SharedSync for state --- src/libstore/store-api.cc | 2 +- src/libstore/store-api.hh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 2c4dee518..21fa2939d 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -920,7 +920,7 @@ StorePathSet Store::exportReferences(const StorePathSet & storePaths, const Stor const Store::Stats & Store::getStats() { { - auto state_(state.lock()); + auto state_(state.read()); stats.pathInfoCacheSize = state_->pathInfoCache.size(); } return stats; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 7d5f533c5..8288cfdf0 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -201,7 +201,7 @@ protected: LRUCache pathInfoCache; }; - Sync state; + SharedSync state; std::shared_ptr diskCache; From d9ba2a1634865b2d7306b97902ce5accb89a9bb2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Jul 2024 19:06:49 +0200 Subject: [PATCH 068/142] Fix error message --- src/libfetchers/git-utils.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index ecc71ae47..1a64acc50 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -131,7 +131,7 @@ T peelObject(git_repository * repo, git_object * obj, git_object_t type) T obj2; if (git_object_peel((git_object * *) (typename T::pointer *) Setter(obj2), obj, type)) { auto err = git_error_last(); - throw Error("peeling Git object '%s': %s", git_object_id(obj), err->message); + throw Error("peeling Git object '%s': %s", *git_object_id(obj), err->message); } return obj2; } From 06b686b62dae4db86bb4c449c3418e271dff1b68 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Jul 2024 20:24:58 +0200 Subject: [PATCH 069/142] Handle tarballs that don't consist of a single top-level directory Fixes #4785 (top-level directories are no longer merged into one). Fixes #10983 (top-level non-directories are no longer discarded). --- src/libfetchers/git-utils.cc | 77 ++++++++++++++++++++++++++---------- tests/functional/tarball.sh | 25 ++++++++++++ 2 files changed, 82 insertions(+), 20 deletions(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 1a64acc50..032d8e0bd 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -126,7 +126,7 @@ Object lookupObject(git_repository * repo, const git_oid & oid, git_object_t typ } template -T peelObject(git_repository * repo, git_object * obj, git_object_t type) +T peelObject(git_object * obj, git_object_t type) { T obj2; if (git_object_peel((git_object * *) (typename T::pointer *) Setter(obj2), obj, type)) { @@ -136,6 +136,29 @@ T peelObject(git_repository * repo, git_object * obj, git_object_t type) return obj2; } +template +T dupObject(typename T::pointer obj) +{ + T obj2; + if (git_object_dup((git_object * *) (typename T::pointer *) Setter(obj2), (git_object *) obj)) + throw Error("duplicating object '%s': %s", *git_object_id((git_object *) obj), git_error_last()->message); + return obj2; +} + +/** + * Peel the specified object (i.e. follow tag and commit objects) to + * either a blob or a tree. + */ +static Object peelToTreeOrBlob(git_object * obj) +{ + /* git_object_peel() doesn't handle blob objects, so handle those + specially. */ + if (git_object_type(obj) == GIT_OBJECT_BLOB) + return dupObject(obj); + else + return peelObject(obj, GIT_OBJECT_TREE); +} + struct GitRepoImpl : GitRepo, std::enable_shared_from_this { /** Location of the repository on disk. */ @@ -166,7 +189,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this std::unordered_set done; std::queue todo; - todo.push(peelObject(*this, lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT)); + todo.push(peelObject(lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT)); while (auto commit = pop(todo)) { if (!done.insert(*git_commit_id(commit->get())).second) continue; @@ -184,7 +207,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this uint64_t getLastModified(const Hash & rev) override { - auto commit = peelObject(*this, lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT); + auto commit = peelObject(lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT); return git_commit_time(commit.get()); } @@ -476,11 +499,11 @@ ref GitRepo::openRepo(const std::filesystem::path & path, bool create, struct GitSourceAccessor : SourceAccessor { ref repo; - Tree root; + Object root; GitSourceAccessor(ref repo_, const Hash & rev) : repo(repo_) - , root(peelObject(*repo, lookupObject(*repo, hashToOID(rev)).get(), GIT_OBJECT_TREE)) + , root(peelToTreeOrBlob(lookupObject(*repo, hashToOID(rev)).get())) { } @@ -506,7 +529,7 @@ struct GitSourceAccessor : SourceAccessor std::optional maybeLstat(const CanonPath & path) override { if (path.isRoot()) - return Stat { .type = tDirectory }; + return Stat { .type = git_object_type(root.get()) == GIT_OBJECT_TREE ? tDirectory : tRegular }; auto entry = lookup(path); if (!entry) @@ -616,10 +639,10 @@ struct GitSourceAccessor : SourceAccessor std::optional lookupTree(const CanonPath & path) { if (path.isRoot()) { - Tree tree; - if (git_tree_dup(Setter(tree), root.get())) - throw Error("duplicating directory '%s': %s", showPath(path), git_error_last()->message); - return tree; + if (git_object_type(root.get()) == GIT_OBJECT_TREE) + return dupObject((git_tree *) &*root); + else + return std::nullopt; } auto entry = lookup(path); @@ -646,10 +669,10 @@ struct GitSourceAccessor : SourceAccessor std::variant getTree(const CanonPath & path) { if (path.isRoot()) { - Tree tree; - if (git_tree_dup(Setter(tree), root.get())) - throw Error("duplicating directory '%s': %s", showPath(path), git_error_last()->message); - return tree; + if (git_object_type(root.get()) == GIT_OBJECT_TREE) + return dupObject((git_tree *) &*root); + else + throw Error("Git root object '%s' is not a directory", *git_object_id(root.get())); } auto entry = need(path); @@ -669,6 +692,9 @@ struct GitSourceAccessor : SourceAccessor Blob getBlob(const CanonPath & path, bool expectSymlink) { + if (!expectSymlink && git_object_type(root.get()) == GIT_OBJECT_BLOB) + return dupObject((git_blob *) &*root); + auto notExpected = [&]() { throw Error( @@ -782,8 +808,6 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink std::vector pendingDirs; - size_t componentsToStrip = 1; - void pushBuilder(std::string name) { git_treebuilder * b; @@ -839,9 +863,6 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink { std::span pathComponents2{pathComponents}; - if (pathComponents2.size() <= componentsToStrip) return false; - pathComponents2 = pathComponents2.subspan(componentsToStrip); - updateBuilders( isDir ? pathComponents2 @@ -964,11 +985,27 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink git_tree_entry_filemode(entry)); } - Hash sync() override { + Hash sync() override + { updateBuilders({}); auto [oid, _name] = popBuilder(); + /* If the root directory contains a single entry that is a + directory or a non-executable regular file, return that as + the top-level object. We don't do this for executables + because they don't have a tree hash in the Git object + model. */ + auto _tree = lookupObject(*repo, oid, GIT_OBJECT_TREE); + auto tree = (const git_tree *) &*_tree; + + if (git_tree_entrycount(tree) == 1) { + auto entry = git_tree_entry_byindex(tree, 0); + auto mode = git_tree_entry_filemode(entry); + if (mode == GIT_FILEMODE_BLOB || mode == GIT_FILEMODE_TREE) + oid = *git_tree_entry_id(entry); + } + return toHash(oid); } }; diff --git a/tests/functional/tarball.sh b/tests/functional/tarball.sh index ab357ac78..6799831ce 100755 --- a/tests/functional/tarball.sh +++ b/tests/functional/tarball.sh @@ -83,3 +83,28 @@ path="$(nix flake prefetch --json "tarball+file://$(pwd)/tree.tar.gz" | jq -r .s [[ $(cat "$path/a/zzz") = bar ]] [[ $(cat "$path/c/aap") = bar ]] [[ $(cat "$path/fnord") = bar ]] + +# Test a tarball that has multiple top-level directories. +rm -rf "$TEST_ROOT/tar_root" +mkdir -p "$TEST_ROOT/tar_root" "$TEST_ROOT/tar_root/foo" "$TEST_ROOT/tar_root/bar" +tar cvf "$TEST_ROOT/tar.tar" -C "$TEST_ROOT/tar_root" . +path="$(nix flake prefetch --json "tarball+file://$TEST_ROOT/tar.tar" | jq -r .storePath)" +[[ -d "$path/foo" ]] +[[ -d "$path/bar" ]] + +# Test a tarball that has a single non-executable regular file. +rm -rf "$TEST_ROOT/tar_root" +mkdir -p "$TEST_ROOT/tar_root" +echo bar > "$TEST_ROOT/tar_root/foo" +tar cvf "$TEST_ROOT/tar.tar" -C "$TEST_ROOT/tar_root" . +path="$(nix flake prefetch --refresh --json "tarball+file://$TEST_ROOT/tar.tar" | jq -r .storePath)" +[[ $(cat "$path") = bar ]] + +# Test a tarball that has a single executable regular file. +rm -rf "$TEST_ROOT/tar_root" +mkdir -p "$TEST_ROOT/tar_root" +echo bar > "$TEST_ROOT/tar_root/foo" +chmod +x "$TEST_ROOT/tar_root/foo" +tar cvf "$TEST_ROOT/tar.tar" -C "$TEST_ROOT/tar_root" . +path="$(nix flake prefetch --refresh --json "tarball+file://$TEST_ROOT/tar.tar" | jq -r .storePath)" +[[ $(cat "$path/foo") = bar ]] From b88950ec777ab55a5e924c0b3758bad06a6f0f33 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Jul 2024 20:34:04 +0200 Subject: [PATCH 070/142] Update fetchTree docs --- src/libexpr/primops/fetchTree.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index e59d7fe67..9a3b81bc9 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -559,11 +559,11 @@ static RegisterPrimOp primop_fetchTarball({ .doc = R"( Download the specified URL, unpack it and return the path of the unpacked tree. The file must be a tape archive (`.tar`) compressed - with `gzip`, `bzip2` or `xz`. The top-level path component of the - files in the tarball is removed, so it is best if the tarball - contains a single directory at top level. The typical use of the - function is to obtain external Nix expression dependencies, such as - a particular version of Nixpkgs, e.g. + with `gzip`, `bzip2` or `xz`. If the tarball consists of a + single directory or non-executable file, then the top-level path + component of the files in the tarball is removed. The typical + use of the function is to obtain external Nix expression + dependencies, such as a particular version of Nixpkgs, e.g. ```nix with import (fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz) {}; From 5e83c0427f701df33a37a7993945d7db99f6267e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Jul 2024 20:46:07 +0200 Subject: [PATCH 071/142] Fix test --- tests/unit/libfetchers/git-utils.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/libfetchers/git-utils.cc b/tests/unit/libfetchers/git-utils.cc index d3547ec6a..f0d38d50c 100644 --- a/tests/unit/libfetchers/git-utils.cc +++ b/tests/unit/libfetchers/git-utils.cc @@ -103,7 +103,7 @@ TEST_F(GitUtilsTest, sink_hardlink) sink->createHardlink(CanonPath("foo-1.1/link"), CanonPath("hello")); FAIL() << "Expected an exception"; } catch (const nix::Error & e) { - ASSERT_THAT(e.msg(), testing::HasSubstr("invalid hard link target")); + ASSERT_THAT(e.msg(), testing::HasSubstr("cannot find hard link target")); ASSERT_THAT(e.msg(), testing::HasSubstr("/hello")); ASSERT_THAT(e.msg(), testing::HasSubstr("foo-1.1/link")); } From 6af40f488a26fda5de55e16c6ba950c6f99762ee Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 27 Jul 2024 01:38:18 +0200 Subject: [PATCH 072/142] Rename SyncBase::read() -> readLock() Make it explicit so it's clear what it's about when I and other contributors read its call sites. --- src/libstore/store-api.cc | 2 +- src/libutil/posix-source-accessor.cc | 2 +- src/libutil/sync.hh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 21fa2939d..b3e5ad014 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -920,7 +920,7 @@ StorePathSet Store::exportReferences(const StorePathSet & storePaths, const Stor const Store::Stats & Store::getStats() { { - auto state_(state.read()); + auto state_(state.readLock()); stats.pathInfoCacheSize = state_->pathInfoCache.size(); } return stats; diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 35047f89e..2b1a485d5 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -97,7 +97,7 @@ std::optional PosixSourceAccessor::cachedLstat(const CanonPath & pa Path absPath = makeAbsPath(path).string(); { - auto cache(_cache.read()); + auto cache(_cache.readLock()); auto i = cache->find(absPath); if (i != cache->end()) return i->second; } diff --git a/src/libutil/sync.hh b/src/libutil/sync.hh index c1b699ffc..d340f3d97 100644 --- a/src/libutil/sync.hh +++ b/src/libutil/sync.hh @@ -106,7 +106,7 @@ public: * Acquire read access to the inner value. When using * `std::shared_mutex`, this will use a shared lock. */ - ReadLock read() const { return ReadLock(const_cast(this)); } + ReadLock readLock() const { return ReadLock(const_cast(this)); } }; template From 22f943bb1fe3feefdf4e469270de79406a46f5c3 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 27 Jul 2024 02:10:31 +0200 Subject: [PATCH 073/142] dependencies: Centralize aws-sdk-cpp and sync with Nixpkgs By syncing with Nixpkgs, we reuse the same derivation, which is generally a good idea, and has the benefit that it is transitively a channel blocker. Changes: - https://github.com/NixOS/nixpkgs/pull/163313 (SuperSandro2000) > nix: disable big-parallel for aws-sdk-cpp > aws-sdk-cpp only takes ~1m52s on a 4 core machine under 50% load > which does not justify the requirement on big parallel. > Tested with `nix-build -A nixVersions.nix_2_6.aws-sdk-cpp`. > I can finally build nix without requiring a big-parallel machine. - https://github.com/NixOS/nixpkgs/pull/227506 (Artturin) > nix: use [ ] instead null to empty requiredSystemFeatures > fixes 'error: value is null while a list was expected' with 'nixpkgs.hostPlatform.gcc.arch = "x86_64";' --- package.nix | 5 +---- packaging/dependencies.nix | 9 +++++++++ src/libstore/package.nix | 5 +---- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/package.nix b/package.nix index c3e565399..2499d4370 100644 --- a/package.nix +++ b/package.nix @@ -237,10 +237,7 @@ in { ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid # There have been issues building these dependencies ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin)) - (aws-sdk-cpp.override { - apis = ["s3" "transfer"]; - customMemoryManagement = false; - }) + aws-sdk-cpp ; propagatedBuildInputs = [ diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index b77e1d14f..e0737593f 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -79,6 +79,15 @@ scope: { inherit stdenv versionSuffix; version = lib.fileContents ../.version + versionSuffix; + aws-sdk-cpp = (pkgs.aws-sdk-cpp.override { + apis = [ "s3" "transfer" ]; + customMemoryManagement = false; + }).overrideAttrs { + # only a stripped down version is built, which takes a lot less resources + # to build, so we don't need a "big-parallel" machine. + requiredSystemFeatures = [ ]; + }; + libseccomp = pkgs.libseccomp.overrideAttrs (_: rec { version = "2.5.5"; src = pkgs.fetchurl { diff --git a/src/libstore/package.nix b/src/libstore/package.nix index 02ff4194a..4582ba0d2 100644 --- a/src/libstore/package.nix +++ b/src/libstore/package.nix @@ -66,10 +66,7 @@ mkMesonDerivation (finalAttrs: { ] ++ lib.optional stdenv.hostPlatform.isLinux libseccomp # There have been issues building these dependencies ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin)) - (aws-sdk-cpp.override { - apis = ["s3" "transfer"]; - customMemoryManagement = false; - }) + aws-sdk-cpp ; propagatedBuildInputs = [ From 17b5d404457018ab1634fbbc38211607682ee37a Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 27 Jul 2024 02:39:55 +0200 Subject: [PATCH 074/142] package.nix: Empty build inputs if not doBuild --- package.nix | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/package.nix b/package.nix index 2499d4370..a7c8923e8 100644 --- a/package.nix +++ b/package.nix @@ -216,7 +216,8 @@ in { ] ++ lib.optional stdenv.hostPlatform.isStatic unixtools.hexdump ; - buildInputs = lib.optionals doBuild [ + buildInputs = lib.optionals doBuild ( + [ brotli bzip2 curl @@ -238,12 +239,13 @@ in { # There have been issues building these dependencies ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin)) aws-sdk-cpp - ; + ); - propagatedBuildInputs = [ + propagatedBuildInputs = lib.optionals doBuild ([ boost nlohmann_json - ] ++ lib.optional enableGC boehmgc; + ] ++ lib.optional enableGC boehmgc + ); dontBuild = !attrs.doBuild; doCheck = attrs.doCheck; From f4464873f514a1aa05ca43178dfca6e1c978f0c9 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 27 Jul 2024 13:01:56 +0200 Subject: [PATCH 075/142] tests/nixos/remote-builds: Print hello world to stderr Trying to learn more about enigmatic spurious hang at https://hydra.nixos.org/build/267517233/nixlog/8 - builder1 seems to have started properly - ssh connection and session are established - ssh client doesn't exit or client.succeed does not return for some reason. Seeing the stdout on the console might give a tiny bit more info. --- tests/nixos/remote-builds.nix | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/nixos/remote-builds.nix b/tests/nixos/remote-builds.nix index 1661203ec..8813832f5 100644 --- a/tests/nixos/remote-builds.nix +++ b/tests/nixos/remote-builds.nix @@ -104,7 +104,10 @@ in builder.succeed("mkdir -p -m 700 /root/.ssh") builder.copy_from_host("key.pub", "/root/.ssh/authorized_keys") builder.wait_for_unit("sshd") - client.succeed(f"ssh -o StrictHostKeyChecking=no {builder.name} 'echo hello world'") + client.succeed(f""" + ssh -o StrictHostKeyChecking=no {builder.name} \ + 'echo hello world on $(hostname)' >&2 + """) # Perform a build and check that it was performed on the builder. out = client.succeed( From 7c5a0b06a4a7908560bca273049e561c0f8193b2 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 27 Jul 2024 13:08:30 +0200 Subject: [PATCH 076/142] tests/nixos/remote-builds: Wait for multi-user This should make the test more robust, considering the strange hang in https://hydra.nixos.org/build/267517233/nixlog/8 `builder` seems to have reached `multi-user.target` before the SSH connection was established, but this seems to be coincidental. This does tell us that enforcing this has a minimal cost in terms of runtime. Waiting for `multi-user.target` on the client is honestly paranoid, but flaky tests are very bad for productivity. --- tests/nixos/remote-builds.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/nixos/remote-builds.nix b/tests/nixos/remote-builds.nix index 8813832f5..8ddf6ad02 100644 --- a/tests/nixos/remote-builds.nix +++ b/tests/nixos/remote-builds.nix @@ -104,6 +104,11 @@ in builder.succeed("mkdir -p -m 700 /root/.ssh") builder.copy_from_host("key.pub", "/root/.ssh/authorized_keys") builder.wait_for_unit("sshd") + # Make sure the builder can handle our login correctly + builder.wait_for_unit("multi-user.target") + # Make sure there's no funny business on the client either + # (should not be necessary, but we have reason to be careful) + client.wait_for_unit("multi-user.target") client.succeed(f""" ssh -o StrictHostKeyChecking=no {builder.name} \ 'echo hello world on $(hostname)' >&2 From cc5b8cdc858467a759405ff8bba074bb9464b3b8 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 27 Jul 2024 13:42:03 +0200 Subject: [PATCH 077/142] buildNoTests: Disable unit tests This seems to have been the intent all along. The odd combination of unit tests, but no functional tests caused a build error where some data for the unit test was source-filtered out. Apparently. It's unclear to me why that happened, so I'm proposing this alternate "fix" to get the buildNoTests to pass. It would be nice to test more configurations, but this mode of building is on the way out anyway, so let's just make it pass and see what configurations make sense to test as part of the meson migration. --- flake.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/flake.nix b/flake.nix index 2384c2974..674c5ae8c 100644 --- a/flake.nix +++ b/flake.nix @@ -143,6 +143,7 @@ nix_noTests = final.nix.override { doInstallCheck = false; + doCheck = false; }; # See https://github.com/NixOS/nixpkgs/pull/214409 From aa2b1d10e29b3839760f02b1e30e70d70cb44ed2 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 27 Jul 2024 14:58:57 +0200 Subject: [PATCH 078/142] Update doc/manual/rl-next/10734-nix3-build-show-all-fod-errors-with-keep-going.md --- .../10734-nix3-build-show-all-fod-errors-with-keep-going.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/rl-next/10734-nix3-build-show-all-fod-errors-with-keep-going.md b/doc/manual/rl-next/10734-nix3-build-show-all-fod-errors-with-keep-going.md index e4e2f797c..1d623e952 100644 --- a/doc/manual/rl-next/10734-nix3-build-show-all-fod-errors-with-keep-going.md +++ b/doc/manual/rl-next/10734-nix3-build-show-all-fod-errors-with-keep-going.md @@ -3,7 +3,7 @@ synopsis: "nix3-build: show all FOD errors with `--keep-going`" prs: 10734 --- -The [`nix build`](@docroot@/command-ref/new-cli/nix3-build.md) command has been updated to improve the behavior of the [`--keep-going`] flag. Now, when `--keep-going` is used, all hash-mismatch errors of failing fixed-output derivations (FODs) are displayed, similar to the behavior of `nix build`. This enhancement ensures that all relevant build errors are shown, making it easier for users to update multiple derivations at once or to diagnose and fix issues. +The [`nix build`](@docroot@/command-ref/new-cli/nix3-build.md) command has been updated to improve the behavior of the [`--keep-going`] flag. Now, when `--keep-going` is used, all hash-mismatch errors of failing fixed-output derivations (FODs) are displayed, similar to the behavior for other build failures. This enhancement ensures that all relevant build errors are shown, making it easier for users to update multiple derivations at once or to diagnose and fix issues. Author: [**Jörg Thalheim (@Mic92)**](https://github.com/Mic92), [**Maximilian Bosch (@Ma27)**](https://github.com/Ma27) From e0198c513accf7f3b481ca71ff482840f294a1da Mon Sep 17 00:00:00 2001 From: Pino Toscano Date: Sun, 28 Jul 2024 11:40:16 +0200 Subject: [PATCH 079/142] libcmd: do not compile editline helpers when building w/ readline The internal "completionCallback" and "listPossibleCallback" helpers are used only when building with editline; hence, do not build then when using readline, matching their usage in "ReadlineLikeInteracter::init()". --- src/libcmd/repl-interacter.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc index 76fe38780..187af46ea 100644 --- a/src/libcmd/repl-interacter.cc +++ b/src/libcmd/repl-interacter.cc @@ -35,6 +35,7 @@ void sigintHandler(int signo) static detail::ReplCompleterMixin * curRepl; // ugly +#ifndef USE_READLINE static char * completionCallback(char * s, int * match) { auto possible = curRepl->completePrefix(s); @@ -101,6 +102,7 @@ static int listPossibleCallback(char * s, char *** avp) return ac; } +#endif ReadlineLikeInteracter::Guard ReadlineLikeInteracter::init(detail::ReplCompleterMixin * repl) { From 1b47748e5a839c714864d51c9ec1ce86ea353456 Mon Sep 17 00:00:00 2001 From: Ivan Trubach Date: Sun, 28 Jul 2024 13:08:33 +0300 Subject: [PATCH 080/142] libstore: return ENOTSUP for getxattr functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change updates the seccomp profile to return ENOTSUP for getxattr functions family. This reflects the behavior of filesystems that don’t support extended attributes (or have an option to disable them), e.g. ext2. The current behavior is confusing for some programs because we can read extended attributes, but only get to know that they are not supported when setting them. In addition to that, ACLs on Linux are implemented via extended attributes internally and if we don’t return ENOTSUP, acl library converts file mode to ACL. https://git.savannah.nongnu.org/cgit/acl.git/tree/libacl/acl_get_file.c?id=d9bb1759d4dad2f28a6dcc8c1742ff75d16dd10d#n69 --- src/libstore/unix/build/local-derivation-goal.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 364e9a148..d3482df17 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -1702,10 +1702,13 @@ void setupSeccomp() throw SysError("unable to add seccomp rule"); } - /* Prevent builders from creating EAs or ACLs. Not all filesystems + /* Prevent builders from using EAs or ACLs. Not all filesystems support these, and they're not allowed in the Nix store because they're not representable in the NAR serialisation. */ - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 || + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(getxattr), 0) != 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lgetxattr), 0) != 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fgetxattr), 0) != 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 || seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 || seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0) throw SysError("unable to add seccomp rule"); From 933f2c086a96d8ce358cc787cf06c50a07263e9e Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Sun, 28 Jul 2024 15:34:48 +0200 Subject: [PATCH 081/142] docs: fix link to building instructions (#11207) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ab647e53b..021e54a3b 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Full reference documentation can be found in the [Nix manual](https://nix.dev/re ## Building and developing -Follow instructions in the Nix reference manual to [set up a development environment and build Nix from source](https://nix.dev/manual/nix/development/development/building.html). +Follow instructions in the Nix reference manual to [set up a development environment and build Nix from source](https://nix.dev/manual/nix/development/building.html). ## Contributing From 96e06b2b06266684d5d74f5f809bdd82ebeb7b04 Mon Sep 17 00:00:00 2001 From: Pino Toscano Date: Sun, 28 Jul 2024 11:31:28 +0200 Subject: [PATCH 082/142] libutil: remove template id from constructors This is not allowed in C++20, and GCC 14 warns about it: ../src/libutil/ref.hh:26:20: warning: template-id not allowed for constructor in C++20 [-Wtemplate-id-cdtor] 26 | explicit ref(const std::shared_ptr & p) | ^ ../src/libutil/ref.hh:26:20: note: remove the '< >' ../src/libutil/ref.hh:33:21: warning: template-id not allowed for constructor in C++20 [-Wtemplate-id-cdtor] 33 | explicit ref(T * p) | ^ ../src/libutil/ref.hh:33:21: note: remove the '< >' --- src/libutil/ref.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libutil/ref.hh b/src/libutil/ref.hh index 016fdd74a..3d0e64ab4 100644 --- a/src/libutil/ref.hh +++ b/src/libutil/ref.hh @@ -23,14 +23,14 @@ public: : p(r.p) { } - explicit ref(const std::shared_ptr & p) + explicit ref(const std::shared_ptr & p) : p(p) { if (!p) throw std::invalid_argument("null pointer cast to ref"); } - explicit ref(T * p) + explicit ref(T * p) : p(p) { if (!p) From c34077578ea48af7534f4dd2aaeef39cb64d9814 Mon Sep 17 00:00:00 2001 From: Pino Toscano Date: Sun, 28 Jul 2024 17:33:24 +0200 Subject: [PATCH 083/142] libutil: fix/improve includes in current-process.cc - move from a __linux__ block to a !_WIN32 block: this matches what the actual code does, using getrlimit() & setrlimit() in !_WIN32 blocks - drop , which is not portable, and it is not used --- src/libutil/current-process.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc index 6ca48220d..c2b1ac500 100644 --- a/src/libutil/current-process.cc +++ b/src/libutil/current-process.cc @@ -15,13 +15,12 @@ #if __linux__ # include -# include # include "cgroup.hh" # include "namespaces.hh" #endif #ifndef _WIN32 -# include +# include #endif namespace nix { From e0012b97abb4c97ccf7fb20299d7b62dd906e89d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 29 Jul 2024 14:26:25 +0200 Subject: [PATCH 084/142] Split tarball-specific logic from GitFileSystemObjectSink --- src/libfetchers/git-utils.cc | 33 ++++++++++++++++------------- src/libfetchers/git-utils.hh | 11 ++++++++++ src/libfetchers/github.cc | 5 +++-- src/libfetchers/tarball.cc | 6 ++++-- tests/unit/libfetchers/git-utils.cc | 2 +- 5 files changed, 37 insertions(+), 20 deletions(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 032d8e0bd..c2d4fe6aa 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -486,6 +486,24 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this return narHash; } + + Hash dereferenceSingletonDirectory(const Hash & oid_) override + { + auto oid = hashToOID(oid_); + + /* If the root directory contains */ + auto _tree = lookupObject(*this, oid, GIT_OBJECT_TREE); + auto tree = (const git_tree *) &*_tree; + + if (git_tree_entrycount(tree) == 1) { + auto entry = git_tree_entry_byindex(tree, 0); + auto mode = git_tree_entry_filemode(entry); + if (mode == GIT_FILEMODE_BLOB || mode == GIT_FILEMODE_TREE) + oid = *git_tree_entry_id(entry); + } + + return toHash(oid); + } }; ref GitRepo::openRepo(const std::filesystem::path & path, bool create, bool bare) @@ -991,21 +1009,6 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink auto [oid, _name] = popBuilder(); - /* If the root directory contains a single entry that is a - directory or a non-executable regular file, return that as - the top-level object. We don't do this for executables - because they don't have a tree hash in the Git object - model. */ - auto _tree = lookupObject(*repo, oid, GIT_OBJECT_TREE); - auto tree = (const git_tree *) &*_tree; - - if (git_tree_entrycount(tree) == 1) { - auto entry = git_tree_entry_byindex(tree, 0); - auto mode = git_tree_entry_filemode(entry); - if (mode == GIT_FILEMODE_BLOB || mode == GIT_FILEMODE_TREE) - oid = *git_tree_entry_id(entry); - } - return toHash(oid); } }; diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh index 495916f62..644f22a07 100644 --- a/src/libfetchers/git-utils.hh +++ b/src/libfetchers/git-utils.hh @@ -98,6 +98,17 @@ struct GitRepo * serialisation. This is memoised on-disk. */ virtual Hash treeHashToNarHash(const Hash & treeHash) = 0; + + /** + * If the specified Git object is a directory with a single entry + * that is a directory or a non-executable regular file, return + * the ID of that object. + * + * Note: We don't do this for executable files because they don't + * have a tree hash in the Git object model that distinguishes + * them from non-executable files. + */ + virtual Hash dereferenceSingletonDirectory(const Hash & oid) = 0; }; ref getTarballCache(); diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 2968d2df2..a2ac9247a 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -255,11 +255,12 @@ struct GitArchiveInputScheme : InputScheme }); TarArchive archive { *source }; - auto parseSink = getTarballCache()->getFileSystemObjectSink(); + auto tarballCache = getTarballCache(); + auto parseSink = tarballCache->getFileSystemObjectSink(); auto lastModified = unpackTarfileToSink(archive, *parseSink); TarballInfo tarballInfo { - .treeHash = parseSink->sync(), + .treeHash = tarballCache->dereferenceSingletonDirectory(parseSink->sync()), .lastModified = lastModified }; diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 55db3eafb..b09f628a4 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -164,7 +164,8 @@ DownloadTarballResult downloadTarball( TarArchive{path}; }) : TarArchive{*source}; - auto parseSink = getTarballCache()->getFileSystemObjectSink(); + auto tarballCache = getTarballCache(); + auto parseSink = tarballCache->getFileSystemObjectSink(); auto lastModified = unpackTarfileToSink(archive, *parseSink); auto res(_res->lock()); @@ -177,7 +178,8 @@ DownloadTarballResult downloadTarball( infoAttrs = cached->value; } else { infoAttrs.insert_or_assign("etag", res->etag); - infoAttrs.insert_or_assign("treeHash", parseSink->sync().gitRev()); + infoAttrs.insert_or_assign("treeHash", + tarballCache->dereferenceSingletonDirectory(parseSink->sync()).gitRev()); infoAttrs.insert_or_assign("lastModified", uint64_t(lastModified)); if (res->immutableUrl) infoAttrs.insert_or_assign("immutableUrl", *res->immutableUrl); diff --git a/tests/unit/libfetchers/git-utils.cc b/tests/unit/libfetchers/git-utils.cc index f0d38d50c..de5110cc3 100644 --- a/tests/unit/libfetchers/git-utils.cc +++ b/tests/unit/libfetchers/git-utils.cc @@ -77,7 +77,7 @@ TEST_F(GitUtilsTest, sink_basic) // sink->createHardlink("foo-1.1/links/foo-2", CanonPath("foo-1.1/hello")); - auto result = sink->sync(); + auto result = repo->dereferenceSingletonDirectory(sink->sync()); auto accessor = repo->getAccessor(result, false); auto entries = accessor->readDirectory(CanonPath::root); ASSERT_EQ(entries.size(), 5); From 7c18b4d0600bfb8ad4159712025f1dcb790aa36f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 29 Jul 2024 14:34:02 +0200 Subject: [PATCH 085/142] Don't dereference top-level regular files Since this yielded an empty directory as far back as Nix 2.3, we don't really need special handling for executables vs non-executables. --- src/libfetchers/git-utils.cc | 3 +-- src/libfetchers/git-utils.hh | 7 +------ tests/functional/tarball.sh | 10 +--------- 3 files changed, 3 insertions(+), 17 deletions(-) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index c2d4fe6aa..114aa4ec0 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -491,14 +491,13 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this { auto oid = hashToOID(oid_); - /* If the root directory contains */ auto _tree = lookupObject(*this, oid, GIT_OBJECT_TREE); auto tree = (const git_tree *) &*_tree; if (git_tree_entrycount(tree) == 1) { auto entry = git_tree_entry_byindex(tree, 0); auto mode = git_tree_entry_filemode(entry); - if (mode == GIT_FILEMODE_BLOB || mode == GIT_FILEMODE_TREE) + if (mode == GIT_FILEMODE_TREE) oid = *git_tree_entry_id(entry); } diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh index 644f22a07..ca37f2e80 100644 --- a/src/libfetchers/git-utils.hh +++ b/src/libfetchers/git-utils.hh @@ -101,12 +101,7 @@ struct GitRepo /** * If the specified Git object is a directory with a single entry - * that is a directory or a non-executable regular file, return - * the ID of that object. - * - * Note: We don't do this for executable files because they don't - * have a tree hash in the Git object model that distinguishes - * them from non-executable files. + * that is a directory, return the ID of that object. */ virtual Hash dereferenceSingletonDirectory(const Hash & oid) = 0; }; diff --git a/tests/functional/tarball.sh b/tests/functional/tarball.sh index 6799831ce..4d8945625 100755 --- a/tests/functional/tarball.sh +++ b/tests/functional/tarball.sh @@ -92,15 +92,7 @@ path="$(nix flake prefetch --json "tarball+file://$TEST_ROOT/tar.tar" | jq -r .s [[ -d "$path/foo" ]] [[ -d "$path/bar" ]] -# Test a tarball that has a single non-executable regular file. -rm -rf "$TEST_ROOT/tar_root" -mkdir -p "$TEST_ROOT/tar_root" -echo bar > "$TEST_ROOT/tar_root/foo" -tar cvf "$TEST_ROOT/tar.tar" -C "$TEST_ROOT/tar_root" . -path="$(nix flake prefetch --refresh --json "tarball+file://$TEST_ROOT/tar.tar" | jq -r .storePath)" -[[ $(cat "$path") = bar ]] - -# Test a tarball that has a single executable regular file. +# Test a tarball that has a single regular file. rm -rf "$TEST_ROOT/tar_root" mkdir -p "$TEST_ROOT/tar_root" echo bar > "$TEST_ROOT/tar_root/foo" From 71865dee2d2818a0955e784901de9c93a1f2baf7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 29 Jul 2024 15:04:55 +0200 Subject: [PATCH 086/142] Fix fetchTarball docs --- src/libexpr/primops/fetchTree.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 9a3b81bc9..f79b6b7b8 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -560,10 +560,10 @@ static RegisterPrimOp primop_fetchTarball({ Download the specified URL, unpack it and return the path of the unpacked tree. The file must be a tape archive (`.tar`) compressed with `gzip`, `bzip2` or `xz`. If the tarball consists of a - single directory or non-executable file, then the top-level path - component of the files in the tarball is removed. The typical - use of the function is to obtain external Nix expression - dependencies, such as a particular version of Nixpkgs, e.g. + single directory, then the top-level path component of the files + in the tarball is removed. The typical use of the function is to + obtain external Nix expression dependencies, such as a + particular version of Nixpkgs, e.g. ```nix with import (fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz) {}; From e8bf2e74a5361033104670ab7148e6f8843073af Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 29 Jul 2024 15:09:06 +0200 Subject: [PATCH 087/142] Add release note --- doc/manual/rl-next/tarball-fixes.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 doc/manual/rl-next/tarball-fixes.md diff --git a/doc/manual/rl-next/tarball-fixes.md b/doc/manual/rl-next/tarball-fixes.md new file mode 100644 index 000000000..c938e9db6 --- /dev/null +++ b/doc/manual/rl-next/tarball-fixes.md @@ -0,0 +1,9 @@ +--- +synopsis: "Improve handling of tarballs that don't consist of a single top-level directory" +prs: +- 11195 +--- + +In previous Nix releases, the tarball fetcher (used by `builtins.fetchTarball`) erroneously merged top-level directories into a single directory, and silently discarded top-level files that are not directories. This is no longer the case. The new behaviour is that *only* if the tarball consists of a single directory, the top-level path component of the files in the tarball is removed (similar to `tar`'s `--strip-components=1`). + +Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) From a3171cec541a8df3be19e1a5a60e7cd4ae978ef9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 29 Jul 2024 15:12:01 +0200 Subject: [PATCH 088/142] Update src/libfetchers/git-utils.hh Co-authored-by: Robert Hensing --- src/libfetchers/git-utils.hh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh index ca37f2e80..915252868 100644 --- a/src/libfetchers/git-utils.hh +++ b/src/libfetchers/git-utils.hh @@ -102,6 +102,7 @@ struct GitRepo /** * If the specified Git object is a directory with a single entry * that is a directory, return the ID of that object. + * Otherwise, return the passed ID unchanged. */ virtual Hash dereferenceSingletonDirectory(const Hash & oid) = 0; }; From 12717325ccb5afeb686c8ddd02f9c9b623021f86 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 29 Jul 2024 12:16:05 -0400 Subject: [PATCH 089/142] Make sure we use `-isystem` with Meson on some deps Otherwise we get warnings on external code. --- src/libexpr/meson.build | 8 +++++++- src/libexpr/primops/fromTOML.cc | 3 --- src/libstore/meson.build | 3 ++- src/libstore/s3-binary-cache-store.cc | 3 --- src/libutil/meson.build | 1 + 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index fa90e7b41..4d8a38b43 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -32,6 +32,7 @@ subdir('build-utils-meson/threads') boost = dependency( 'boost', modules : ['container', 'context'], + include_type: 'system', ) # boost is a public dependency, but not a pkg-config dependency unfortunately, so we # put in `deps_other`. @@ -55,7 +56,12 @@ if bdw_gc.found() endif configdata.set('HAVE_BOEHMGC', bdw_gc.found().to_int()) -toml11 = dependency('toml11', version : '>=3.7.0', method : 'cmake') +toml11 = dependency( + 'toml11', + version : '>=3.7.0', + method : 'cmake', + include_type: 'system', +) deps_other += toml11 config_h = configure_file( diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 70755f9e0..b4f1df7a8 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -3,10 +3,7 @@ #include -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wswitch-enum" #include -#pragma GCC diagnostic pop namespace nix { diff --git a/src/libstore/meson.build b/src/libstore/meson.build index cb8110f3f..50b15e15d 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -73,6 +73,7 @@ subdir('build-utils-meson/threads') boost = dependency( 'boost', modules : ['container'], + include_type: 'system', ) # boost is a public dependency, but not a pkg-config dependency unfortunately, so we # put in `deps_other`. @@ -113,7 +114,7 @@ if aws_s3.found() '-laws-cpp-sdk-core', '-laws-crt-cpp', ], - ) + ).as_system('system') endif deps_other += aws_s3 diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 1082657bb..92ab47cd6 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -10,8 +10,6 @@ #include "compression.hh" #include "filetransfer.hh" -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wswitch-enum" #include #include #include @@ -27,7 +25,6 @@ #include #include #include -#pragma GCC diagnostic pop using namespace Aws::Transfer; diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 04c778c31..8552c4c9d 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -62,6 +62,7 @@ endif boost = dependency( 'boost', modules : ['context', 'coroutine'], + include_type: 'system', ) # boost is a public dependency, but not a pkg-config dependency unfortunately, so we # put in `deps_other`. From f380becffacaabcfbace6b00e640cdc7bd0bbf64 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 29 Jul 2024 23:25:31 +0200 Subject: [PATCH 090/142] Credit all contributors in release notes --- .../data/release-credits-email-to-handle.json | 51 +++++ .../data/release-credits-handle-to-name.json | 44 +++++ maintainers/release-credits | 176 ++++++++++++++++++ maintainers/release-notes | 7 + maintainers/release-process.md | 4 + 5 files changed, 282 insertions(+) create mode 100644 maintainers/data/release-credits-email-to-handle.json create mode 100644 maintainers/data/release-credits-handle-to-name.json create mode 100755 maintainers/release-credits diff --git a/maintainers/data/release-credits-email-to-handle.json b/maintainers/data/release-credits-email-to-handle.json new file mode 100644 index 000000000..573ec2b31 --- /dev/null +++ b/maintainers/data/release-credits-email-to-handle.json @@ -0,0 +1,51 @@ +{ + "bogus": "bogus", + "edolstra@gmail.com": "edolstra", + "roberth@users.noreply.github.com": "roberth", + "toscano.pino@tiscali.it": "pinotree", + "valentin@gagarin.work": "fricklerhandwerk", + "mr.trubach@icloud.com": "tie", + "robert@roberthensing.nl": "roberth", + "lix@jade.fyi": "lf-", + "cole.e.helbling@outlook.com": "cole-h", + "joerg@thalheim.io": "Mic92", + "John.Ericson@Obsidian.Systems": "Ericson2314", + "ryan.hendrickson@alum.mit.edu": "rhendric", + "67135060+poweredbypie@users.noreply.github.com": "poweredbypie", + "detroyejr@outlook.com": "detroyejr", + "silvan.mosberger@tweag.io": "infinisil", + "vcs@emily.moe": "emilazy", + "farid.m.zakaria@gmail.com": "fzakaria", + "22859658+RTUnreal@users.noreply.github.com": "RTUnreal", + "me@las.rs": "L-as", + "philip.taron@gmail.com": "philiptaron", + "root@goldstein.rs": "GoldsteinE", + "tomberek@users.noreply.github.com": "tomberek", + "lexi.mattick@neuralink.com": "kognise", + "andrew@johnandrewmarshall.com": "amarshall", + "contact@romain-neil.fr": "romain-neil", + "Mic92@users.noreply.github.com": "Mic92", + "valentin.gagarin@tweag.io": "fricklerhandwerk", + "siddhantk232@gmail.com": "siddhantk232", + "kn@openbsd.org": "klemensn", + "slyich@gmail.com": "trofi", + "theophane.hufschmitt@tweag.io": "thufschmitt", + "alicebob@lijzij.de": "alicebob", + "winter@winter.cafe": "winterqt", + "brian@brianmckenna.org": "puffnfresh", + "git@haenoe.party": "haenoe", + "peshogo@gmail.com": "pineapplehunter", + "poweredbypie@users.noreply.github.com": "poweredbypie", + "arthur200126@gmail.com": "Artoria2e5", + "tomberek@gmail.com": "tomberek", + "jaredbaur@fastmail.com": "jmbaur", + "andreas@rammhold.de": "andir", + "hamirmahal@gmail.com": "hamirmahal", + "git@JohnEricson.me": "Ericson2314", + "8763518+SkamDart@users.noreply.github.com": "SkamDart", + "kirillrdy@gmail.com": "kirillrdy", + "pennae@lix.systems": "pennae", + "delroth@gmail.com": "delroth", + "enno@nerdworks.de": "elohmeier", + "mjbauer95@gmail.com": "matthewbauer" +} \ No newline at end of file diff --git a/maintainers/data/release-credits-handle-to-name.json b/maintainers/data/release-credits-handle-to-name.json new file mode 100644 index 000000000..d68311dde --- /dev/null +++ b/maintainers/data/release-credits-handle-to-name.json @@ -0,0 +1,44 @@ +{ + "fzakaria": "Farid Zakaria", + "kognise": "Lexi Mattick", + "L-as": "Las Safin", + "haenoe": "HaeNoe", + "andir": "Andreas Rammhold", + "matthewbauer": "Matthew Bauer", + "emilazy": "Emily", + "pineapplehunter": "Shogo Takata", + "RTUnreal": null, + "jmbaur": "Jared Baur", + "Ericson2314": "John Ericson", + "pinotree": "Pino Toscano", + "tie": "Ivan Trubach", + "poweredbypie": null, + "fricklerhandwerk": "Valentin Gagarin", + "Mic92": "J\u00f6rg Thalheim", + "alicebob": "Harmen", + "elohmeier": "Enno Richter", + "delroth": "Pierre Bourdon", + "kirillrdy": null, + "thufschmitt": "Th\u00e9ophane Hufschmitt", + "detroyejr": "Jonathan De Troye", + "klemensn": "Klemens Nanni", + "tomberek": null, + "rhendric": "Ryan Hendrickson", + "philiptaron": "Philip Taron", + "puffnfresh": "Brian McKenna", + "lf-": "jade", + "romain-neil": "Romain Neil", + "hamirmahal": "Hamir Mahal", + "edolstra": "Eelco Dolstra", + "Artoria2e5": "Mingye Wang", + "SkamDart": "Cameron", + "roberth": "Robert Hensing", + "amarshall": "Andrew Marshall", + "trofi": "Sergei Trofimovich", + "cole-h": "Cole Helbling", + "infinisil": "Silvan Mosberger", + "siddhantk232": "Siddhant Kumar", + "winterqt": "Winter", + "GoldsteinE": "Max \u201cGoldstein\u201d Siling", + "pennae": null +} \ No newline at end of file diff --git a/maintainers/release-credits b/maintainers/release-credits new file mode 100755 index 000000000..8350c4e2a --- /dev/null +++ b/maintainers/release-credits @@ -0,0 +1,176 @@ +#!/usr/bin/env nix +#!nix develop --impure --expr +#!nix `` +#!nix let flake = builtins.getFlake ("git+file://" + toString ../.); +#!nix pkgs = flake.inputs.nixpkgs.legacyPackages.${builtins.currentSystem}; +#!nix in pkgs.mkShell { nativeBuildInputs = [ +#!nix (pkgs.python3.withPackages (ps: with ps; [ requests ])) +#!nix ]; } +#!nix `` --command python3 + +# This script lists out the contributors for a given release. +# It must be run from the root of the Nix repository. + +import os +import sys +import json +import requests + +github_token = os.environ.get("GITHUB_TOKEN") +if not github_token: + print("GITHUB_TOKEN is not set. If you hit the rate limit, set it", file=sys.stderr) + # Might be ok, as we have a cache. + # raise ValueError("GITHUB_TOKEN must be set") + +# 1. Read the current version in .version +version = os.environ.get("VERSION") +if not version: + version = open(".version").read().strip() + +print(f"Generating release credits for Nix {version}", file=sys.stderr) + +# 2. Compute previous version +vcomponents = version.split(".") +if len(vcomponents) >= 2: + prev_version = f"{vcomponents[0]}.{int(vcomponents[1])-1}.0" +else: + raise ValueError(".version must have at least two components") + +# For unreleased versions +endref = "HEAD" +# For older releases +# endref = version + +# 2. Find the merge base between the current version and the previous version +mergeBase = os.popen(f"git merge-base {prev_version} {endref}").read().strip() +print(f"Merge base between {prev_version} and {endref} is {mergeBase}", file=sys.stderr) + +# 3. Find the date of the merge base +mergeBaseDate = os.popen(f"git show -s --format=%ci {mergeBase}").read().strip()[0:10] +print(f"Merge base date is {mergeBaseDate}", file=sys.stderr) + +# 4. Get the commits between the merge base and the current version + +def get_commits(): + raw = os.popen(f"git log --pretty=format:'%H\t%an\t%ae' {mergeBase}..{endref}").read().strip() + lines = raw.split("\n") + return [ { "hash": items[0], "author": items[1], "email": items[2] } + for line in lines + for items in (line.split("\t"),) + ] + +def commits_to_first_commit_by_email(commits): + by_email = dict() + for commit in commits: + email = commit["email"] + if email not in by_email: + by_email[email] = commit + return by_email + + +samples = commits_to_first_commit_by_email(get_commits()) + +# For quick testing, only pick two samples from the dict +# samples = dict(list(samples.items())[:2]) + +# Query the GitHub API to get handle +def get_github_commit(commit): + url = f"https://api.github.com/repos/NixOS/nix/commits/{commit['hash']}" + headers = {'Authorization': f'token {github_token}'} + response = requests.get(url, headers=headers) + response.raise_for_status() + return response.json() + +class Cache: + def __init__(self, filename, require = True): + self.filename = filename + try: + with open(filename, "r") as f: + self.values = json.load(f) + except FileNotFoundError: + if require: + raise + self.values = dict() + def save(self): + with open(self.filename, "w") as f: + json.dump(self.values, f, indent=4) + print(f"Saved cache to {self.filename}", file=sys.stderr) + +# The email to handle cache maps email addresses to either +# - a handle (string) +# - None (if no handle was found) +email_to_handle_cache = Cache("maintainers/data/release-credits-email-to-handle.json") + +handles = set() +emails = dict() + +for sample in samples: + s = samples[sample] + email = s["email"] + if not email in email_to_handle_cache.values: + print(f"Querying GitHub API for {s['hash']}, to get handle for {s['email']}") + ghc = get_github_commit(samples[sample]) + gha = ghc["author"] + if gha and gha["login"]: + handle = gha["login"] + print(f"Handle: {handle}") + email_to_handle_cache.values[email] = handle + else: + print(f"Found no handle for {s['email']}") + email_to_handle_cache.values[email] = None + handle = email_to_handle_cache.values[email] + if handle is not None: + handles.add(handle) + else: + emails[email] = s["author"] + +# print(email_to_handle_cache.values) + +email_to_handle_cache.save() + +handle_to_name_cache = Cache("maintainers/data/release-credits-handle-to-name.json") + +print(f"Found {len(handles)} handles", file=sys.stderr) + +for handle in handles: + if not handle in handle_to_name_cache.values: + print(f"Querying GitHub API for {handle}, to get name", file=sys.stderr) + url = f"https://api.github.com/users/{handle}" + headers = {'Authorization': f'token {github_token}'} + response = requests.get(url, headers=headers) + response.raise_for_status() + user = response.json() + name = user["name"] + print(f"Name: {name}", file=sys.stderr) + handle_to_name_cache.values[handle] = name + +handle_to_name_cache.save() + +entries = list() + +for handle in handles: + name = handle_to_name_cache.values[handle] + if name is None: + # This way it looks more regular + name = handle + entries += [ f"- {name} [**(@{handle})**](https://github.com/{handle})" ] + +def shuffle(entries): + salt = os.urandom(16) + return sorted(entries, key=lambda x: hash((x, salt))) + +# Fair ordering is undecidable +entries = shuffle(entries) + +# For a sanity check, we could sort the entries by handle instead. +# entries = sorted(entries) + +print("") +print(f"This release was made possible by the following {len(entries)} contributors:") +print("") + +for entry in entries: + print(entry) + +for email in emails: + print(f"- {emails[email]}") diff --git a/maintainers/release-notes b/maintainers/release-notes index 0fca5abf2..b7b5731c8 100755 --- a/maintainers/release-notes +++ b/maintainers/release-notes @@ -151,6 +151,13 @@ section_title="Release $version_full ($DATE)" echo "# $section_title" echo changelog-d doc/manual/rl-next | sed -e 's/ *$//' + + if ! $IS_PATCH; then + echo + echo "# Contributors" + echo + VERSION=$version_full ./maintainers/release-credits + fi ) | tee -a $file log "Wrote $file" diff --git a/maintainers/release-process.md b/maintainers/release-process.md index da6886ea9..7a2b3c0a7 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -39,6 +39,10 @@ release: * Proof-read / edit / rearrange the release notes if needed. Breaking changes and highlights should go to the top. +* Run `maintainers/release-credits` to make sure the credits script works + and produces a sensible output. Some emails might not automatically map to + a GitHub handle. + * Push. ```console From 850e4779d6285feffd78dbf4c12eb203a3f10637 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 30 Jul 2024 00:15:49 +0200 Subject: [PATCH 091/142] release-credits: Remove bots --- maintainers/release-credits | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/maintainers/release-credits b/maintainers/release-credits index 8350c4e2a..18641a07b 100755 --- a/maintainers/release-credits +++ b/maintainers/release-credits @@ -148,12 +148,19 @@ handle_to_name_cache.save() entries = list() +def is_bot(handle): + return handle.endswith("[bot]") + for handle in handles: name = handle_to_name_cache.values[handle] if name is None: # This way it looks more regular name = handle - entries += [ f"- {name} [**(@{handle})**](https://github.com/{handle})" ] + + # Note to bots: I'm only doing this because it would distract humans, and + # therefore delay your creation. I'm sorry; it's for your benefit. + if not is_bot(handle): + entries += [ f"- {name} [**(@{handle})**](https://github.com/{handle})" ] def shuffle(entries): salt = os.urandom(16) From d7f46cf28ea521eef0530a1a41af56a34dd89278 Mon Sep 17 00:00:00 2001 From: Pino Toscano Date: Tue, 30 Jul 2024 05:29:32 +0200 Subject: [PATCH 092/142] makefiles: recognize GNU/Hurd Set HOST_HURD & HOST_UNIX for GNU/Hurd in the makefile-based build system; the latter variable is important as it will include all the commit Unix bits. --- mk/platform.mk | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mk/platform.mk b/mk/platform.mk index fe960dedf..22c114a20 100644 --- a/mk/platform.mk +++ b/mk/platform.mk @@ -29,4 +29,8 @@ ifdef HOST_OS HOST_SOLARIS = 1 HOST_UNIX = 1 endif + ifeq ($(HOST_KERNEL), gnu) + HOST_HURD = 1 + HOST_UNIX = 1 + endif endif From 7442f4a16143fe69a71949f1edbbbcab6862ed23 Mon Sep 17 00:00:00 2001 From: Pino Toscano Date: Tue, 30 Jul 2024 05:31:42 +0200 Subject: [PATCH 093/142] libutil: use /proc/self/exe on Hurd as well Rely on the Linux-compatible procfs available on the Hurd to get the path of the current executable. --- src/libutil/current-process.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc index c2b1ac500..0bc46d746 100644 --- a/src/libutil/current-process.cc +++ b/src/libutil/current-process.cc @@ -137,7 +137,7 @@ std::optional getSelfExe() { static auto cached = []() -> std::optional { - #if __linux__ + #if __linux__ || __GNU__ return readLink("/proc/self/exe"); #elif __APPLE__ char buf[1024]; From a1ccf6061396f398526bb99518cfd3b9c432aeb1 Mon Sep 17 00:00:00 2001 From: Pino Toscano Date: Tue, 30 Jul 2024 05:34:34 +0200 Subject: [PATCH 094/142] tests: define fallback PATH_MAX Few filesystem-related tests rely on PATH_MAX for buffers, and PATH_MAX is optional in POSIX (and not available on the Hurd). To make them build and pass, provide a fallback definition of PATH_MAX in case not available. Ideally speaking, the tests ought to not unconditionally rely on PATH_MAX, do alternative strategies (e.g. dynamically allocate buffers, expand them as needed, etc); OTOH this is test code, so it would be more work that what it would be worth, so IMHO the define fallback is good enough. --- tests/unit/libutil/tests.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/unit/libutil/tests.cc b/tests/unit/libutil/tests.cc index 8a3ca8561..2b73d323b 100644 --- a/tests/unit/libutil/tests.cc +++ b/tests/unit/libutil/tests.cc @@ -17,6 +17,10 @@ # define FS_ROOT FS_SEP #endif +#ifndef PATH_MAX +# define PATH_MAX 4096 +#endif + namespace nix { /* ----------- tests for util.hh ------------------------------------------------*/ From ee86e7f361c55c8c7dc2e45c3868802af249aeff Mon Sep 17 00:00:00 2001 From: Corbin Simpson Date: Tue, 30 Jul 2024 05:51:47 -0700 Subject: [PATCH 095/142] doc/command-ref/nix-shell: Shebangs can occur anywhere (#11202) Co-authored-by: Valentin Gagarin --- doc/manual/src/command-ref/nix-shell.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index ddec30f5b..69a711bd5 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -297,3 +297,8 @@ with import {}; runCommand "dummy" { buildInputs = [ python pythonPackages.prettytable ]; } "" ``` + +The script's file name is passed as the first argument to the interpreter specified by the `-i` flag. + +Aside from the very first line, which is a directive to the operating system, the additional `#! nix-shell` lines do not need to be at the beginning of the file. +This allows wrapping them in block comments for languages where `#` does not start a comment, such as ECMAScript, Erlang, PHP, or Ruby. From f011cfd28dc359bb786990d94b63c2c482e1b8da Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 30 Jul 2024 17:35:46 +0200 Subject: [PATCH 096/142] maintainers/release-*: Add mode line This lets various tools figure out the language more easily. --- maintainers/release-credits | 1 + maintainers/release-notes | 1 + 2 files changed, 2 insertions(+) diff --git a/maintainers/release-credits b/maintainers/release-credits index 18641a07b..7a5c87d7d 100755 --- a/maintainers/release-credits +++ b/maintainers/release-credits @@ -1,4 +1,5 @@ #!/usr/bin/env nix +# vim: set filetype=python: #!nix develop --impure --expr #!nix `` #!nix let flake = builtins.getFlake ("git+file://" + toString ../.); diff --git a/maintainers/release-notes b/maintainers/release-notes index b7b5731c8..c0c4ee734 100755 --- a/maintainers/release-notes +++ b/maintainers/release-notes @@ -1,4 +1,5 @@ #!/usr/bin/env nix +# vim: set filetype=bash: #!nix shell .#changelog-d --command bash # --- CONFIGURATION --- From ef8021744855754a12a4b0d343d902b2a8a29d8c Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Mon, 29 Jul 2024 10:42:43 -0400 Subject: [PATCH 097/142] fix: add flake headers --- src/libflake/flake/nix-flake.pc.in | 10 ++++++++++ src/libflake/local.mk | 5 +++++ src/nix/meson.build | 1 + 3 files changed, 16 insertions(+) create mode 100644 src/libflake/flake/nix-flake.pc.in diff --git a/src/libflake/flake/nix-flake.pc.in b/src/libflake/flake/nix-flake.pc.in new file mode 100644 index 000000000..10c52f5e9 --- /dev/null +++ b/src/libflake/flake/nix-flake.pc.in @@ -0,0 +1,10 @@ +prefix=@prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Nix +Description: Nix Package Manager +Version: @PACKAGE_VERSION@ +Requires: nix-util nix-store nix-expr +Libs: -L${libdir} -lnixflake +Cflags: -I${includedir}/nix -std=c++2a diff --git a/src/libflake/local.mk b/src/libflake/local.mk index 2cceda2bf..5e604ef3a 100644 --- a/src/libflake/local.mk +++ b/src/libflake/local.mk @@ -15,3 +15,8 @@ libflake_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetcher libflake_LDFLAGS += $(THREAD_LDFLAGS) libflake_LIBS = libutil libstore libfetchers libexpr + +$(eval $(call install-file-in, $(buildprefix)$(d)/flake/nix-flake.pc, $(libdir)/pkgconfig, 0644)) + +$(foreach i, $(wildcard src/libflake/flake/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644))) diff --git a/src/nix/meson.build b/src/nix/meson.build index 53bb083a9..7405548de 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -20,6 +20,7 @@ deps_private_maybe_subproject = [ dependency('nix-util'), dependency('nix-store'), dependency('nix-expr'), + dependency('nix-flake'), dependency('nix-fetchers'), dependency('nix-main'), dependency('nix-cmd'), From dd75711895057328b00f92353730e7f873eafd52 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Fri, 12 Jul 2024 14:56:30 +0200 Subject: [PATCH 098/142] Use std::strong_ordering for version comparison The actual motive here is the avoidance of integer overflow if we were to make these use checked NixInts and retain the subtraction. However, the actual *intent* of this code is a three-way comparison, which can be done with operator<=>, so we should just do *that* instead. Change-Id: I7f9a7da1f3176424b528af6d1b4f1591e4ab26bf --- src/libexpr/primops.cc | 3 ++- src/libstore/names.cc | 8 ++++---- src/libstore/names.hh | 2 +- src/nix-env/nix-env.cc | 22 +++++++++++----------- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 0b3b19b57..a23583f34 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -4446,7 +4446,8 @@ static void prim_compareVersions(EvalState & state, const PosIdx pos, Value * * { auto version1 = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.compareVersions"); auto version2 = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.compareVersions"); - v.mkInt(compareVersions(version1, version2)); + auto result = compareVersions(version1, version2); + v.mkInt(result < 0 ? -1 : result > 0 ? 1 : 0); } static RegisterPrimOp primop_compareVersions({ diff --git a/src/libstore/names.cc b/src/libstore/names.cc index 277aabf0f..c0e1b1022 100644 --- a/src/libstore/names.cc +++ b/src/libstore/names.cc @@ -94,7 +94,7 @@ static bool componentsLT(const std::string_view c1, const std::string_view c2) } -int compareVersions(const std::string_view v1, const std::string_view v2) +std::strong_ordering compareVersions(const std::string_view v1, const std::string_view v2) { auto p1 = v1.begin(); auto p2 = v2.begin(); @@ -102,11 +102,11 @@ int compareVersions(const std::string_view v1, const std::string_view v2) while (p1 != v1.end() || p2 != v2.end()) { auto c1 = nextComponent(p1, v1.end()); auto c2 = nextComponent(p2, v2.end()); - if (componentsLT(c1, c2)) return -1; - else if (componentsLT(c2, c1)) return 1; + if (componentsLT(c1, c2)) return std::strong_ordering::less; + else if (componentsLT(c2, c1)) return std::strong_ordering::greater; } - return 0; + return std::strong_ordering::equal; } diff --git a/src/libstore/names.hh b/src/libstore/names.hh index d82b99bb4..a6909d545 100644 --- a/src/libstore/names.hh +++ b/src/libstore/names.hh @@ -30,7 +30,7 @@ typedef std::list DrvNames; std::string_view nextComponent(std::string_view::const_iterator & p, const std::string_view::const_iterator end); -int compareVersions(const std::string_view v1, const std::string_view v2); +std::strong_ordering compareVersions(const std::string_view v1, const std::string_view v2); DrvNames drvNamesFromArgs(const Strings & opArgs); } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 40c200542..ba2baccee 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -204,15 +204,15 @@ static void loadDerivations(EvalState & state, const SourcePath & nixExprPath, } -static long getPriority(EvalState & state, PackageInfo & drv) +static NixInt getPriority(EvalState & state, PackageInfo & drv) { - return drv.queryMetaInt("priority", 0); + return drv.queryMetaInt("priority", NixInt(0)); } -static long comparePriorities(EvalState & state, PackageInfo & drv1, PackageInfo & drv2) +static std::strong_ordering comparePriorities(EvalState & state, PackageInfo & drv1, PackageInfo & drv2) { - return getPriority(state, drv2) - getPriority(state, drv1); + return getPriority(state, drv2) <=> getPriority(state, drv1); } @@ -280,7 +280,7 @@ std::vector pickNewestOnly(EvalState & state, std::vector matches) auto & oneDrv = match.packageInfo; const auto drvName = DrvName { oneDrv.queryName() }; - long comparison = 1; + std::strong_ordering comparison = std::strong_ordering::greater; const auto itOther = newest.find(drvName.name); @@ -288,9 +288,9 @@ std::vector pickNewestOnly(EvalState & state, std::vector matches) auto & newestDrv = itOther->second.packageInfo; comparison = - oneDrv.querySystem() == newestDrv.querySystem() ? 0 : - oneDrv.querySystem() == settings.thisSystem ? 1 : - newestDrv.querySystem() == settings.thisSystem ? -1 : 0; + oneDrv.querySystem() == newestDrv.querySystem() ? std::strong_ordering::equal : + oneDrv.querySystem() == settings.thisSystem ? std::strong_ordering::greater : + newestDrv.querySystem() == settings.thisSystem ? std::strong_ordering::less : std::strong_ordering::equal; if (comparison == 0) comparison = comparePriorities(state, oneDrv, newestDrv); if (comparison == 0) @@ -625,13 +625,13 @@ static void upgradeDerivations(Globals & globals, continue; DrvName newName(j->queryName()); if (newName.name == drvName.name) { - int d = compareVersions(drvName.version, newName.version); + std::strong_ordering d = compareVersions(drvName.version, newName.version); if ((upgradeType == utLt && d < 0) || (upgradeType == utLeq && d <= 0) || (upgradeType == utEq && d == 0) || upgradeType == utAlways) { - long d2 = -1; + std::strong_ordering d2 = std::strong_ordering::less; if (bestElem != availElems.end()) { d2 = comparePriorities(*globals.state, *bestElem, *j); if (d2 == 0) d2 = compareVersions(bestVersion, newName.version); @@ -902,7 +902,7 @@ static VersionDiff compareVersionAgainstSet( for (auto & i : elems) { DrvName name2(i.queryName()); if (name.name == name2.name) { - int d = compareVersions(name.version, name2.version); + std::strong_ordering d = compareVersions(name.version, name2.version); if (d < 0) { diff = cvGreater; version = name2.version; From e28cb67d41643bf2073e46149e6f5d10d1fd0ef5 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Fri, 5 Jul 2024 15:52:55 +0200 Subject: [PATCH 099/142] libutil: add checked arithmetic tools This is in preparation for adding checked arithmetic to the evaluator. Change-Id: I6e115ce8f5411feda1706624977a4dcd5efd4d13 --- src/libutil/checked-arithmetic.hh | 182 ++++++++++++++++++ src/libutil/meson.build | 1 + .../tests/gtest-with-params.hh | 54 ++++++ tests/unit/libutil/checked-arithmetic.cc | 158 +++++++++++++++ 4 files changed, 395 insertions(+) create mode 100644 src/libutil/checked-arithmetic.hh create mode 100644 tests/unit/libutil-support/tests/gtest-with-params.hh create mode 100644 tests/unit/libutil/checked-arithmetic.cc diff --git a/src/libutil/checked-arithmetic.hh b/src/libutil/checked-arithmetic.hh new file mode 100644 index 000000000..55d6ad205 --- /dev/null +++ b/src/libutil/checked-arithmetic.hh @@ -0,0 +1,182 @@ +#pragma once +/** + * @file Checked arithmetic with classes that make it hard to accidentally make something an unchecked operation. + */ + +#include +#include // IWYU pragma: keep +#include +#include +#include +#include +#include + +namespace nix::checked { + +class DivideByZero : std::exception +{}; + +/** + * Numeric value enforcing checked arithmetic. Performing mathematical operations on such values will return a Result + * type which needs to be checked. + */ +template +struct Checked +{ + using Inner = T; + + // TODO: this must be a "trivial default constructor", which means it + // cannot set the value to NOT DO UB on uninit. + T value; + + Checked() = default; + explicit Checked(T const value) + : value{value} + { + } + Checked(Checked const & other) = default; + Checked(Checked && other) = default; + Checked & operator=(Checked const & other) = default; + + std::strong_ordering operator<=>(Checked const & other) const = default; + std::strong_ordering operator<=>(T const & other) const + { + return value <=> other; + } + + explicit operator T() const + { + return value; + } + + enum class OverflowKind { + NoOverflow, + Overflow, + DivByZero, + }; + + class Result + { + T value; + OverflowKind overflowed_; + + public: + Result(T value, bool overflowed) + : value{value} + , overflowed_{overflowed ? OverflowKind::Overflow : OverflowKind::NoOverflow} + { + } + Result(T value, OverflowKind overflowed) + : value{value} + , overflowed_{overflowed} + { + } + + bool operator==(Result other) const + { + return value == other.value && overflowed_ == other.overflowed_; + } + + std::optional valueChecked() const + { + if (overflowed_ != OverflowKind::NoOverflow) { + return std::nullopt; + } else { + return value; + } + } + + /** + * Returns the result as if the arithmetic were performed as wrapping arithmetic. + * + * \throws DivideByZero if the operation was a divide by zero. + */ + T valueWrapping() const + { + if (overflowed_ == OverflowKind::DivByZero) { + throw DivideByZero{}; + } + return value; + } + + bool overflowed() const + { + return overflowed_ == OverflowKind::Overflow; + } + + bool divideByZero() const + { + return overflowed_ == OverflowKind::DivByZero; + } + }; + + Result operator+(Checked const other) const + { + return (*this) + other.value; + } + Result operator+(T const other) const + { + T result; + bool overflowed = __builtin_add_overflow(value, other, &result); + return Result{result, overflowed}; + } + + Result operator-(Checked const other) const + { + return (*this) - other.value; + } + Result operator-(T const other) const + { + T result; + bool overflowed = __builtin_sub_overflow(value, other, &result); + return Result{result, overflowed}; + } + + Result operator*(Checked const other) const + { + return (*this) * other.value; + } + Result operator*(T const other) const + { + T result; + bool overflowed = __builtin_mul_overflow(value, other, &result); + return Result{result, overflowed}; + } + + Result operator/(Checked const other) const + { + return (*this) / other.value; + } + /** + * Performs a checked division. + * + * If the right hand side is zero, the result is marked as a DivByZero and + * valueWrapping will throw. + */ + Result operator/(T const other) const + { + constexpr T const minV = std::numeric_limits::min(); + + // It's only possible to overflow with signed division since doing so + // requires crossing the two's complement limits by MIN / -1 (since + // two's complement has one more in range in the negative direction + // than in the positive one). + if (std::is_signed() && (value == minV && other == -1)) { + return Result{minV, true}; + } else if (other == 0) { + return Result{0, OverflowKind::DivByZero}; + } else { + T result = value / other; + return Result{result, false}; + } + } +}; + +template +std::ostream & operator<<(std::ostream & ios, Checked v) +{ + ios << v.value; + return ios; +} + +} diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 8552c4c9d..e0299177f 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -171,6 +171,7 @@ headers = [config_h] + files( 'args/root.hh', 'callback.hh', 'canon-path.hh', + 'checked-arithmetic.hh', 'chunked-vector.hh', 'closure.hh', 'comparator.hh', diff --git a/tests/unit/libutil-support/tests/gtest-with-params.hh b/tests/unit/libutil-support/tests/gtest-with-params.hh new file mode 100644 index 000000000..323a083fe --- /dev/null +++ b/tests/unit/libutil-support/tests/gtest-with-params.hh @@ -0,0 +1,54 @@ +#pragma once +// SPDX-FileCopyrightText: 2014 Emil Eriksson +// +// SPDX-License-Identifier: BSD-2-Clause +// +// The lion's share of this code is copy pasted directly out of RapidCheck +// headers, so the copyright is set accordingly. +/** + * @file Implements the ability to run a RapidCheck test under gtest with changed + * test parameters such as the number of tests to run. This is useful for + * running very large numbers of the extremely cheap property tests. + */ + +#include +#include +#include + +namespace rc::detail { + +using MakeTestParams = TestParams (*)(); + +template +void checkGTestWith(Testable && testable, MakeTestParams makeTestParams) +{ + const auto testInfo = ::testing::UnitTest::GetInstance()->current_test_info(); + detail::TestMetadata metadata; + metadata.id = std::string(testInfo->test_case_name()) + "/" + std::string(testInfo->name()); + metadata.description = std::string(testInfo->name()); + + const auto result = checkTestable(std::forward(testable), metadata, makeTestParams()); + + if (result.template is()) { + const auto success = result.template get(); + if (!success.distribution.empty()) { + printResultMessage(result, std::cout); + std::cout << std::endl; + } + } else { + std::ostringstream ss; + printResultMessage(result, ss); + FAIL() << ss.str() << std::endl; + } +} +} + +#define RC_GTEST_PROP_WITH_PARAMS(TestCase, Name, MakeParams, ArgList) \ + void rapidCheck_propImpl_##TestCase##_##Name ArgList; \ + \ + TEST(TestCase, Name) \ + { \ + ::rc::detail::checkGTestWith(&rapidCheck_propImpl_##TestCase##_##Name, MakeParams); \ + } \ + \ + void rapidCheck_propImpl_##TestCase##_##Name ArgList diff --git a/tests/unit/libutil/checked-arithmetic.cc b/tests/unit/libutil/checked-arithmetic.cc new file mode 100644 index 000000000..75018660d --- /dev/null +++ b/tests/unit/libutil/checked-arithmetic.cc @@ -0,0 +1,158 @@ +#include +#include +#include +#include +#include +#include + +#include + +#include "tests/gtest-with-params.hh" + +namespace rc { +using namespace nix; + +template +struct Arbitrary> +{ + static Gen> arbitrary() + { + return gen::arbitrary(); + } +}; + +} + +namespace nix::checked { + +// Pointer to member function! Mildly gross. +template +using Oper = Checked::Result (Checked::*)(T const other) const; + +template +using ReferenceOper = T (*)(T a, T b); + +/** + * Checks that performing an operation that overflows into an inaccurate result + * has the desired behaviour. + * + * TBig is a type large enough to represent all results of TSmall operations. + */ +template +void checkType(TSmall a_, TSmall b, Oper oper, ReferenceOper reference) +{ + // Sufficient to fit all values + TBig referenceResult = reference(a_, b); + constexpr const TSmall minV = std::numeric_limits::min(); + constexpr const TSmall maxV = std::numeric_limits::max(); + + Checked a{a_}; + auto result = (a.*(oper))(b); + + // Just truncate it to get the in-range result + RC_ASSERT(result.valueWrapping() == static_cast(referenceResult)); + + if (referenceResult > maxV || referenceResult < minV) { + RC_ASSERT(result.overflowed()); + RC_ASSERT(!result.valueChecked().has_value()); + } else { + RC_ASSERT(!result.overflowed()); + RC_ASSERT(result.valueChecked().has_value()); + RC_ASSERT(*result.valueChecked() == referenceResult); + } +} + +/** + * Checks that performing an operation that overflows into an inaccurate result + * has the desired behaviour. + * + * TBig is a type large enough to represent all results of TSmall operations. + */ +template +void checkDivision(TSmall a_, TSmall b) +{ + // Sufficient to fit all values + constexpr const TSmall minV = std::numeric_limits::min(); + + Checked a{a_}; + auto result = a / b; + + if (std::is_signed() && a_ == minV && b == -1) { + // This is the only possible overflow condition + RC_ASSERT(result.valueWrapping() == minV); + RC_ASSERT(result.overflowed()); + } else if (b == 0) { + RC_ASSERT(result.divideByZero()); + RC_ASSERT_THROWS_AS(result.valueWrapping(), nix::checked::DivideByZero); + RC_ASSERT(result.valueChecked() == std::nullopt); + } else { + TBig referenceResult = a_ / b; + auto result_ = result.valueChecked(); + RC_ASSERT(result_.has_value()); + RC_ASSERT(*result_ == referenceResult); + RC_ASSERT(result.valueWrapping() == referenceResult); + } +} + +/** Creates parameters that perform a more adequate number of checks to validate + * extremely cheap tests such as arithmetic tests */ +static rc::detail::TestParams makeParams() +{ + auto const & conf = rc::detail::configuration(); + auto newParams = conf.testParams; + newParams.maxSuccess = 10000; + return newParams; +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, add_unsigned, makeParams, (uint16_t a, uint16_t b)) +{ + checkType(a, b, &Checked::operator+, [](int32_t a, int32_t b) { return a + b; }); +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, add_signed, makeParams, (int16_t a, int16_t b)) +{ + checkType(a, b, &Checked::operator+, [](int32_t a, int32_t b) { return a + b; }); +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, sub_unsigned, makeParams, (uint16_t a, uint16_t b)) +{ + checkType(a, b, &Checked::operator-, [](int32_t a, int32_t b) { return a - b; }); +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, sub_signed, makeParams, (int16_t a, int16_t b)) +{ + checkType(a, b, &Checked::operator-, [](int32_t a, int32_t b) { return a - b; }); +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, mul_unsigned, makeParams, (uint16_t a, uint16_t b)) +{ + checkType(a, b, &Checked::operator*, [](int64_t a, int64_t b) { return a * b; }); +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, mul_signed, makeParams, (int16_t a, int16_t b)) +{ + checkType(a, b, &Checked::operator*, [](int64_t a, int64_t b) { return a * b; }); +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, div_unsigned, makeParams, (uint16_t a, uint16_t b)) +{ + checkDivision(a, b); +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, div_signed, makeParams, (int16_t a, int16_t b)) +{ + checkDivision(a, b); +} + +// Make absolutely sure that we check the special cases if the proptest +// generator does not come up with them. This one is especially important +// because it has very specific pairs required for the edge cases unlike the +// others. +TEST(Checked, div_signed_special_cases) +{ + checkDivision(std::numeric_limits::min(), -1); + checkDivision(std::numeric_limits::min(), 0); + checkDivision(0, 0); +} + +} From 7b6622d73317f0cd093cde96b5898b8af07abea7 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Fri, 12 Jul 2024 16:22:34 +0200 Subject: [PATCH 100/142] language: cleanly ban integer overflows This also bans various sneaking of negative numbers from the language into unsuspecting builtins as was exposed while auditing the consequences of changing the Nix language integer type to a newtype. It's unlikely that this change comprehensively ensures correctness when passing integers out of the Nix language and we should probably add a checked-narrowing function or something similar, but that's out of scope for the immediate change. During the development of this I found a few fun facts about the language: - You could overflow integers by converting from unsigned JSON values. - You could overflow unsigned integers by converting negative numbers into them when going into Nix config, into fetchTree, and into flake inputs. The flake inputs and Nix config cannot actually be tested properly since they both ban thunks, however, we put in checks anyway because it's possible these could somehow be used to do such shenanigans some other way. Note that Lix has banned Nix language integer overflows since the very first public beta, but threw a SIGILL about them because we run with -fsanitize=signed-overflow -fsanitize-undefined-trap-on-error in production builds. Since the Nix language uses signed integers, overflow was simply undefined behaviour, and since we defined that to trap, it did. Trapping on it was a bad UX, but we didn't even entirely notice that we had done this at all until it was reported as a bug a couple of months later (which is, to be fair, that flag working as intended), and it's got enough production time that, aside from code that is IMHO buggy (and which is, in any case, not in nixpkgs) such as https://git.lix.systems/lix-project/lix/issues/445, we don't think anyone doing anything reasonable actually depends on wrapping overflow. Even for weird use cases such as doing funny bit crimes, it doesn't make sense IMO to have wrapping behaviour, since two's complement arithmetic overflow behaviour is so *aggressively* not what you want for *any* kind of mathematics/algorithms. The Nix language exists for package management, a domain where bit crimes are already only dubiously in scope to begin with, and it makes a lot more sense for that domain for the integers to never lose precision, either by throwing errors if they would, or by being arbitrary-precision. Fixes: https://github.com/NixOS/nix/issues/10968 Original-CL: https://gerrit.lix.systems/c/lix/+/1596 Change-Id: I51f253840c4af2ea5422b8a420aa5fafbf8fae75 --- src/libcmd/installable-flake.cc | 4 +- src/libcmd/installable-value.hh | 2 +- src/libexpr-c/nix_api_value.cc | 2 +- src/libexpr/eval-cache.cc | 4 +- src/libexpr/eval.cc | 21 ++-- src/libexpr/get-drvs.cc | 4 +- src/libexpr/json-to-value.cc | 7 +- src/libexpr/lexer.l | 2 +- src/libexpr/nixexpr.hh | 1 + src/libexpr/primops.cc | 102 ++++++++++++------ src/libexpr/primops/fetchTree.cc | 12 ++- src/libexpr/value-to-json.cc | 2 +- src/libexpr/value.hh | 10 +- src/libflake/flake/flake.cc | 22 +++- src/libstore/globals.hh | 10 +- .../lang/eval-fail-fetchTree-negative.err.exp | 8 ++ .../lang/eval-fail-fetchTree-negative.nix | 5 + ...ake-ref-to-string-negative-integer.err.exp | 14 +++ ...l-flake-ref-to-string-negative-integer.nix | 7 ++ .../eval-fail-fromJSON-overflowing.err.exp | 8 ++ .../lang/eval-fail-fromJSON-overflowing.nix | 1 + .../lang/eval-fail-overflowing-add.err.exp | 6 ++ .../lang/eval-fail-overflowing-add.nix | 4 + .../lang/eval-fail-overflowing-div.err.exp | 23 ++++ .../lang/eval-fail-overflowing-div.nix | 7 ++ .../lang/eval-fail-overflowing-mul.err.exp | 16 +++ .../lang/eval-fail-overflowing-mul.nix | 3 + .../lang/eval-fail-overflowing-sub.err.exp | 9 ++ .../lang/eval-fail-overflowing-sub.nix | 4 + tests/unit/libexpr-support/tests/libexpr.hh | 2 +- 30 files changed, 260 insertions(+), 62 deletions(-) create mode 100644 tests/functional/lang/eval-fail-fetchTree-negative.err.exp create mode 100644 tests/functional/lang/eval-fail-fetchTree-negative.nix create mode 100644 tests/functional/lang/eval-fail-flake-ref-to-string-negative-integer.err.exp create mode 100644 tests/functional/lang/eval-fail-flake-ref-to-string-negative-integer.nix create mode 100644 tests/functional/lang/eval-fail-fromJSON-overflowing.err.exp create mode 100644 tests/functional/lang/eval-fail-fromJSON-overflowing.nix create mode 100644 tests/functional/lang/eval-fail-overflowing-add.err.exp create mode 100644 tests/functional/lang/eval-fail-overflowing-add.nix create mode 100644 tests/functional/lang/eval-fail-overflowing-div.err.exp create mode 100644 tests/functional/lang/eval-fail-overflowing-div.nix create mode 100644 tests/functional/lang/eval-fail-overflowing-mul.err.exp create mode 100644 tests/functional/lang/eval-fail-overflowing-mul.nix create mode 100644 tests/functional/lang/eval-fail-overflowing-sub.err.exp create mode 100644 tests/functional/lang/eval-fail-overflowing-sub.nix diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index 8796ad5ba..6c9ee6748 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -104,12 +104,12 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() auto drvPath = attr->forceDerivation(); - std::optional priority; + std::optional priority; if (attr->maybeGetAttr(state->sOutputSpecified)) { } else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) { if (auto aPriority = aMeta->maybeGetAttr("priority")) - priority = aPriority->getInt(); + priority = aPriority->getInt().value; } return {{ diff --git a/src/libcmd/installable-value.hh b/src/libcmd/installable-value.hh index 798cb5e1a..60207cd23 100644 --- a/src/libcmd/installable-value.hh +++ b/src/libcmd/installable-value.hh @@ -40,7 +40,7 @@ struct ExtraPathInfoValue : ExtraPathInfo /** * An optional priority for use with "build envs". See Package */ - std::optional priority; + std::optional priority; /** * The attribute path associated with this value. The idea is diff --git a/src/libexpr-c/nix_api_value.cc b/src/libexpr-c/nix_api_value.cc index 845e87935..fa2a9cbe2 100644 --- a/src/libexpr-c/nix_api_value.cc +++ b/src/libexpr-c/nix_api_value.cc @@ -306,7 +306,7 @@ int64_t nix_get_int(nix_c_context * context, const nix_value * value) try { auto & v = check_value_in(value); assert(v.type() == nix::nInt); - return v.integer(); + return v.integer().value; } NIXC_CATCH_ERRS_RES(0); } diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 26352187e..95b484606 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -326,7 +326,7 @@ struct AttrDb case AttrType::Bool: return {{rowId, queryAttribute.getInt(2) != 0}}; case AttrType::Int: - return {{rowId, int_t{queryAttribute.getInt(2)}}}; + return {{rowId, int_t{NixInt{queryAttribute.getInt(2)}}}}; case AttrType::ListOfStrings: return {{rowId, tokenizeString>(queryAttribute.getStr(2), "\t")}}; case AttrType::Missing: @@ -469,7 +469,7 @@ Value & AttrCursor::forceValue() else if (v.type() == nBool) cachedValue = {root->db->setBool(getKey(), v.boolean()), v.boolean()}; else if (v.type() == nInt) - cachedValue = {root->db->setInt(getKey(), v.integer()), int_t{v.integer()}}; + cachedValue = {root->db->setInt(getKey(), v.integer().value), int_t{v.integer()}}; else if (v.type() == nAttrs) ; // FIXME: do something? else diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 746ccab2a..76c54936f 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1979,7 +1979,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) NixStringContext context; std::vector s; size_t sSize = 0; - NixInt n = 0; + NixInt n{0}; NixFloat nf = 0; bool first = !forceString; @@ -2023,17 +2023,22 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) if (firstType == nInt) { if (vTmp.type() == nInt) { - n += vTmp.integer(); + auto newN = n + vTmp.integer(); + if (auto checked = newN.valueChecked(); checked.has_value()) { + n = NixInt(*checked); + } else { + state.error("integer overflow in adding %1% + %2%", n, vTmp.integer()).atPos(i_pos).debugThrow(); + } } else if (vTmp.type() == nFloat) { // Upgrade the type from int to float; firstType = nFloat; - nf = n; + nf = n.value; nf += vTmp.fpoint(); } else state.error("cannot add %1% to an integer", showType(vTmp)).atPos(i_pos).withFrame(env, *this).debugThrow(); } else if (firstType == nFloat) { if (vTmp.type() == nInt) { - nf += vTmp.integer(); + nf += vTmp.integer().value; } else if (vTmp.type() == nFloat) { nf += vTmp.fpoint(); } else @@ -2158,7 +2163,7 @@ NixFloat EvalState::forceFloat(Value & v, const PosIdx pos, std::string_view err try { forceValue(v, pos); if (v.type() == nInt) - return v.integer(); + return v.integer().value; else if (v.type() != nFloat) error( "expected a float but found %1%: %2%", @@ -2345,7 +2350,7 @@ BackedStringView EvalState::coerceToString( shell scripting convenience, just like `null'. */ if (v.type() == nBool && v.boolean()) return "1"; if (v.type() == nBool && !v.boolean()) return ""; - if (v.type() == nInt) return std::to_string(v.integer()); + if (v.type() == nInt) return std::to_string(v.integer().value); if (v.type() == nFloat) return std::to_string(v.fpoint()); if (v.type() == nNull) return ""; @@ -2728,9 +2733,9 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v // Special case type-compatibility between float and int if (v1.type() == nInt && v2.type() == nFloat) - return v1.integer() == v2.fpoint(); + return v1.integer().value == v2.fpoint(); if (v1.type() == nFloat && v2.type() == nInt) - return v1.fpoint() == v2.integer(); + return v1.fpoint() == v2.integer().value; // All other types are not compatible with each other. if (v1.type() != v2.type()) return false; diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 7041a3932..20963ec91 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -246,8 +246,8 @@ NixInt PackageInfo::queryMetaInt(const std::string & name, NixInt def) if (v->type() == nString) { /* Backwards compatibility with before we had support for integer meta fields. */ - if (auto n = string2Int(v->c_str())) - return *n; + if (auto n = string2Int(v->c_str())) + return NixInt{*n}; } return def; } diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 21074bdd8..9ac56541a 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -2,6 +2,7 @@ #include "value.hh" #include "eval.hh" +#include #include #include @@ -101,8 +102,12 @@ public: return true; } - bool number_unsigned(number_unsigned_t val) override + bool number_unsigned(number_unsigned_t val_) override { + if (val_ > std::numeric_limits::max()) { + throw Error("unsigned json number %1% outside of Nix integer range", val_); + } + NixInt::Inner val = val_; rs->value(state).mkInt(val); rs->add(); return true; diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index eb1825b7c..a7e44cb72 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -138,7 +138,7 @@ or { return OR_KW; } {INT} { errno = 0; std::optional numMay = string2Int(yytext); if (numMay.has_value()) { - yylval->n = *numMay; + yylval->n = NixInt{*numMay}; } else { throw ParseError(ErrorInfo{ .msg = HintFmt("invalid integer '%1%'", yytext), diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 3279e3d48..7868834f1 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -107,6 +107,7 @@ struct ExprInt : Expr { Value v; ExprInt(NixInt n) { v.mkInt(n); }; + ExprInt(NixInt::Inner n) { v.mkInt(n); }; Value * maybeThunk(EvalState & state, Env & env) override; COMMON_METHODS }; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index a23583f34..acc0d6c1c 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -587,9 +587,9 @@ struct CompareValues { try { if (v1->type() == nFloat && v2->type() == nInt) - return v1->fpoint() < v2->integer(); + return v1->fpoint() < v2->integer().value; if (v1->type() == nInt && v2->type() == nFloat) - return v1->integer() < v2->fpoint(); + return v1->integer().value < v2->fpoint(); if (v1->type() != v2->type()) state.error("cannot compare %s with %s", showType(*v1), showType(*v2)).debugThrow(); // Allow selecting a subset of enum values @@ -2677,13 +2677,13 @@ static struct LazyPosAcessors { PrimOp primop_lineOfPos{ .arity = 1, .fun = [] (EvalState & state, PosIdx pos, Value * * args, Value & v) { - v.mkInt(state.positions[PosIdx(args[0]->integer())].line); + v.mkInt(state.positions[PosIdx(args[0]->integer().value)].line); } }; PrimOp primop_columnOfPos{ .arity = 1, .fun = [] (EvalState & state, PosIdx pos, Value * * args, Value & v) { - v.mkInt(state.positions[PosIdx(args[0]->integer())].column); + v.mkInt(state.positions[PosIdx(args[0]->integer().value)].column); } }; @@ -3159,7 +3159,8 @@ static void elemAt(EvalState & state, const PosIdx pos, Value & list, int n, Val /* Return the n-1'th element of a list. */ static void prim_elemAt(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - elemAt(state, pos, *args[0], state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.elemAt"), v); + NixInt::Inner elem = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.elemAt").value; + elemAt(state, pos, *args[0], elem, v); } static RegisterPrimOp primop_elemAt({ @@ -3453,10 +3454,12 @@ static RegisterPrimOp primop_all({ static void prim_genList(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto len = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.genList"); + auto len_ = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.genList").value; - if (len < 0) - state.error("cannot create list of size %1%", len).atPos(pos).debugThrow(); + if (len_ < 0) + state.error("cannot create list of size %1%", len_).atPos(pos).debugThrow(); + + size_t len = size_t(len_); // More strict than striclty (!) necessary, but acceptable // as evaluating map without accessing any values makes little sense. @@ -3713,9 +3716,17 @@ static void prim_add(EvalState & state, const PosIdx pos, Value * * args, Value if (args[0]->type() == nFloat || args[1]->type() == nFloat) v.mkFloat(state.forceFloat(*args[0], pos, "while evaluating the first argument of the addition") + state.forceFloat(*args[1], pos, "while evaluating the second argument of the addition")); - else - v.mkInt( state.forceInt(*args[0], pos, "while evaluating the first argument of the addition") - + state.forceInt(*args[1], pos, "while evaluating the second argument of the addition")); + else { + auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument of the addition"); + auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument of the addition"); + + auto result_ = i1 + i2; + if (auto result = result_.valueChecked(); result.has_value()) { + v.mkInt(*result); + } else { + state.error("integer overflow in adding %1% + %2%", i1, i2).atPos(pos).debugThrow(); + } + } } static RegisterPrimOp primop_add({ @@ -3734,9 +3745,18 @@ static void prim_sub(EvalState & state, const PosIdx pos, Value * * args, Value if (args[0]->type() == nFloat || args[1]->type() == nFloat) v.mkFloat(state.forceFloat(*args[0], pos, "while evaluating the first argument of the subtraction") - state.forceFloat(*args[1], pos, "while evaluating the second argument of the subtraction")); - else - v.mkInt( state.forceInt(*args[0], pos, "while evaluating the first argument of the subtraction") - - state.forceInt(*args[1], pos, "while evaluating the second argument of the subtraction")); + else { + auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument of the subtraction"); + auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument of the subtraction"); + + auto result_ = i1 - i2; + + if (auto result = result_.valueChecked(); result.has_value()) { + v.mkInt(*result); + } else { + state.error("integer overflow in subtracting %1% - %2%", i1, i2).atPos(pos).debugThrow(); + } + } } static RegisterPrimOp primop_sub({ @@ -3755,9 +3775,18 @@ static void prim_mul(EvalState & state, const PosIdx pos, Value * * args, Value if (args[0]->type() == nFloat || args[1]->type() == nFloat) v.mkFloat(state.forceFloat(*args[0], pos, "while evaluating the first of the multiplication") * state.forceFloat(*args[1], pos, "while evaluating the second argument of the multiplication")); - else - v.mkInt( state.forceInt(*args[0], pos, "while evaluating the first argument of the multiplication") - * state.forceInt(*args[1], pos, "while evaluating the second argument of the multiplication")); + else { + auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument of the multiplication"); + auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument of the multiplication"); + + auto result_ = i1 * i2; + + if (auto result = result_.valueChecked(); result.has_value()) { + v.mkInt(*result); + } else { + state.error("integer overflow in multiplying %1% * %2%", i1, i2).atPos(pos).debugThrow(); + } + } } static RegisterPrimOp primop_mul({ @@ -3784,10 +3813,12 @@ static void prim_div(EvalState & state, const PosIdx pos, Value * * args, Value NixInt i1 = state.forceInt(*args[0], pos, "while evaluating the first operand of the division"); NixInt i2 = state.forceInt(*args[1], pos, "while evaluating the second operand of the division"); /* Avoid division overflow as it might raise SIGFPE. */ - if (i1 == std::numeric_limits::min() && i2 == -1) - state.error("overflow in integer division").atPos(pos).debugThrow(); - - v.mkInt(i1 / i2); + auto result_ = i1 / i2; + if (auto result = result_.valueChecked(); result.has_value()) { + v.mkInt(*result); + } else { + state.error("integer overflow in dividing %1% / %2%", i1, i2).atPos(pos).debugThrow(); + } } } @@ -3802,8 +3833,9 @@ static RegisterPrimOp primop_div({ static void prim_bitAnd(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - v.mkInt(state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitAnd") - & state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitAnd")); + auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitAnd"); + auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitAnd"); + v.mkInt(i1.value & i2.value); } static RegisterPrimOp primop_bitAnd({ @@ -3817,8 +3849,10 @@ static RegisterPrimOp primop_bitAnd({ static void prim_bitOr(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - v.mkInt(state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitOr") - | state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitOr")); + auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitOr"); + auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitOr"); + + v.mkInt(i1.value | i2.value); } static RegisterPrimOp primop_bitOr({ @@ -3832,8 +3866,10 @@ static RegisterPrimOp primop_bitOr({ static void prim_bitXor(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - v.mkInt(state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitXor") - ^ state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitXor")); + auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitXor"); + auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitXor"); + + v.mkInt(i1.value ^ i2.value); } static RegisterPrimOp primop_bitXor({ @@ -3913,13 +3949,19 @@ static RegisterPrimOp primop_toString({ non-negative. */ static void prim_substring(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - int start = state.forceInt(*args[0], pos, "while evaluating the first argument (the start offset) passed to builtins.substring"); + NixInt::Inner start = state.forceInt(*args[0], pos, "while evaluating the first argument (the start offset) passed to builtins.substring").value; if (start < 0) state.error("negative start position in 'substring'").atPos(pos).debugThrow(); - int len = state.forceInt(*args[1], pos, "while evaluating the second argument (the substring length) passed to builtins.substring"); + NixInt::Inner len = state.forceInt(*args[1], pos, "while evaluating the second argument (the substring length) passed to builtins.substring").value; + + // Negative length may be idiomatically passed to builtins.substring to get + // the tail of the string. + if (len < 0) { + len = std::numeric_limits::max(); + } // Special-case on empty substring to avoid O(n) strlen // This allows for the use of empty substrings to efficently capture string context @@ -3962,7 +4004,7 @@ static void prim_stringLength(EvalState & state, const PosIdx pos, Value * * arg { NixStringContext context; auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.stringLength"); - v.mkInt(s->size()); + v.mkInt(NixInt::Inner(s->size())); } static RegisterPrimOp primop_stringLength({ diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index f79b6b7b8..78328701d 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -122,9 +122,15 @@ static void fetchTree( } else if (attr.value->type() == nBool) attrs.emplace(state.symbols[attr.name], Explicit{attr.value->boolean()}); - else if (attr.value->type() == nInt) - attrs.emplace(state.symbols[attr.name], uint64_t(attr.value->integer())); - else if (state.symbols[attr.name] == "publicKeys") { + else if (attr.value->type() == nInt) { + auto intValue = attr.value->integer().value; + + if (intValue < 0) { + state.error("negative value given for fetchTree attr %1%: %2%", state.symbols[attr.name], intValue).atPos(pos).debugThrow(); + } + + attrs.emplace(state.symbols[attr.name], uint64_t(intValue)); + } else if (state.symbols[attr.name] == "publicKeys") { experimentalFeatureSettings.require(Xp::VerifiedFetches); attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, context).dump()); } diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index f8cc05616..8044fe347 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -22,7 +22,7 @@ json printValueAsJSON(EvalState & state, bool strict, switch (v.type()) { case nInt: - out = v.integer(); + out = v.integer().value; break; case nBool: diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 257da1d2d..1510abfea 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -8,6 +8,7 @@ #include "value/context.hh" #include "source-path.hh" #include "print-options.hh" +#include "checked-arithmetic.hh" #if HAVE_BOEHMGC #include @@ -73,8 +74,8 @@ class EvalState; class XMLWriter; class Printer; -typedef int64_t NixInt; -typedef double NixFloat; +using NixInt = checked::Checked; +using NixFloat = double; /** * External values must descend from ExternalValueBase, so that @@ -304,6 +305,11 @@ public: return internalType != tUninitialized; } + inline void mkInt(NixInt::Inner n) + { + mkInt(NixInt{n}); + } + inline void mkInt(NixInt n) { finishValue(tInt, { .integer = n }); diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index 627dfe830..fd1183514 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -140,9 +140,16 @@ static FlakeInput parseFlakeInput(EvalState & state, case nBool: attrs.emplace(state.symbols[attr.name], Explicit { attr.value->boolean() }); break; - case nInt: - attrs.emplace(state.symbols[attr.name], (long unsigned int) attr.value->integer()); + case nInt: { + auto intValue = attr.value->integer().value; + + if (intValue < 0) { + state.error("negative value given for flake input attribute %1%: %2%", state.symbols[attr.name], intValue).debugThrow(); + } + + attrs.emplace(state.symbols[attr.name], uint64_t(intValue)); break; + } default: if (attr.name == state.symbols.create("publicKeys")) { experimentalFeatureSettings.require(Xp::VerifiedFetches); @@ -272,7 +279,7 @@ static Flake readFlake( else if (setting.value->type() == nInt) flake.config.settings.emplace( state.symbols[setting.name], - state.forceInt(*setting.value, setting.pos, "")); + state.forceInt(*setting.value, setting.pos, "").value); else if (setting.value->type() == nBool) flake.config.settings.emplace( state.symbols[setting.name], @@ -904,8 +911,13 @@ static void prim_flakeRefToString( for (const auto & attr : *args[0]->attrs()) { auto t = attr.value->type(); if (t == nInt) { - attrs.emplace(state.symbols[attr.name], - (uint64_t) attr.value->integer()); + auto intValue = attr.value->integer().value; + + if (intValue < 0) { + state.error("negative value given for flake ref attr %1%: %2%", state.symbols[attr.name], intValue).atPos(pos).debugThrow(); + } + + attrs.emplace(state.symbols[attr.name], uint64_t(intValue)); } else if (t == nBool) { attrs.emplace(state.symbols[attr.name], Explicit { attr.value->boolean() }); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 8760c9d14..1ca9f02f5 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -1131,7 +1131,10 @@ public: )"}; Setting maxFree{ - this, std::numeric_limits::max(), "max-free", + // n.b. this is deliberately int64 max rather than uint64 max because + // this goes through the Nix language JSON parser and thus needs to be + // representable in Nix language integers. + this, std::numeric_limits::max(), "max-free", R"( When a garbage collection is triggered by the `min-free` option, it stops as soon as `max-free` bytes are available. The default is @@ -1221,7 +1224,10 @@ public: Setting warnLargePathThreshold{ this, - std::numeric_limits::max(), + // n.b. this is deliberately int64 max rather than uint64 max because + // this goes through the Nix language JSON parser and thus needs to be + // representable in Nix language integers. + std::numeric_limits::max(), "warn-large-path-threshold", R"( Warn when copying a path larger than this number of bytes to the Nix store diff --git a/tests/functional/lang/eval-fail-fetchTree-negative.err.exp b/tests/functional/lang/eval-fail-fetchTree-negative.err.exp new file mode 100644 index 000000000..d9ba1f0b2 --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchTree-negative.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fetchTree' builtin + at /pwd/lang/eval-fail-fetchTree-negative.nix:1:1: + 1| builtins.fetchTree { + | ^ + 2| type = "file"; + + error: negative value given for fetchTree attr owner: -1 diff --git a/tests/functional/lang/eval-fail-fetchTree-negative.nix b/tests/functional/lang/eval-fail-fetchTree-negative.nix new file mode 100644 index 000000000..90bcab5d8 --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchTree-negative.nix @@ -0,0 +1,5 @@ +builtins.fetchTree { + type = "file"; + url = "file://eval-fail-fetchTree-negative.nix"; + owner = -1; +} diff --git a/tests/functional/lang/eval-fail-flake-ref-to-string-negative-integer.err.exp b/tests/functional/lang/eval-fail-flake-ref-to-string-negative-integer.err.exp new file mode 100644 index 000000000..25c8d7eaa --- /dev/null +++ b/tests/functional/lang/eval-fail-flake-ref-to-string-negative-integer.err.exp @@ -0,0 +1,14 @@ +error: + … while calling the 'seq' builtin + at /pwd/lang/eval-fail-flake-ref-to-string-negative-integer.nix:1:16: + 1| let n = -1; in builtins.seq n (builtins.flakeRefToString { + | ^ + 2| type = "github"; + + … while calling the 'flakeRefToString' builtin + at /pwd/lang/eval-fail-flake-ref-to-string-negative-integer.nix:1:32: + 1| let n = -1; in builtins.seq n (builtins.flakeRefToString { + | ^ + 2| type = "github"; + + error: negative value given for flake ref attr repo: -1 diff --git a/tests/functional/lang/eval-fail-flake-ref-to-string-negative-integer.nix b/tests/functional/lang/eval-fail-flake-ref-to-string-negative-integer.nix new file mode 100644 index 000000000..e0208eb25 --- /dev/null +++ b/tests/functional/lang/eval-fail-flake-ref-to-string-negative-integer.nix @@ -0,0 +1,7 @@ +let n = -1; in builtins.seq n (builtins.flakeRefToString { + type = "github"; + owner = "NixOS"; + repo = n; + ref = "23.05"; + dir = "lib"; +}) diff --git a/tests/functional/lang/eval-fail-fromJSON-overflowing.err.exp b/tests/functional/lang/eval-fail-fromJSON-overflowing.err.exp new file mode 100644 index 000000000..a39082b45 --- /dev/null +++ b/tests/functional/lang/eval-fail-fromJSON-overflowing.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fromJSON' builtin + at /pwd/lang/eval-fail-fromJSON-overflowing.nix:1:1: + 1| builtins.fromJSON ''{"attr": 18446744073709551615}'' + | ^ + 2| + + error: unsigned json number 18446744073709551615 outside of Nix integer range diff --git a/tests/functional/lang/eval-fail-fromJSON-overflowing.nix b/tests/functional/lang/eval-fail-fromJSON-overflowing.nix new file mode 100644 index 000000000..6dfbce3f6 --- /dev/null +++ b/tests/functional/lang/eval-fail-fromJSON-overflowing.nix @@ -0,0 +1 @@ +builtins.fromJSON ''{"attr": 18446744073709551615}'' diff --git a/tests/functional/lang/eval-fail-overflowing-add.err.exp b/tests/functional/lang/eval-fail-overflowing-add.err.exp new file mode 100644 index 000000000..6458cf1c9 --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-add.err.exp @@ -0,0 +1,6 @@ +error: integer overflow in adding 9223372036854775807 + 1 + at /pwd/lang/eval-fail-overflowing-add.nix:4:8: + 3| b = 1; + 4| in a + b + | ^ + 5| diff --git a/tests/functional/lang/eval-fail-overflowing-add.nix b/tests/functional/lang/eval-fail-overflowing-add.nix new file mode 100644 index 000000000..24258fc20 --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-add.nix @@ -0,0 +1,4 @@ +let + a = 9223372036854775807; + b = 1; +in a + b diff --git a/tests/functional/lang/eval-fail-overflowing-div.err.exp b/tests/functional/lang/eval-fail-overflowing-div.err.exp new file mode 100644 index 000000000..8ce07d4d6 --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-div.err.exp @@ -0,0 +1,23 @@ +error: + … while calling the 'seq' builtin + at /pwd/lang/eval-fail-overflowing-div.nix:7:4: + 6| b = -1; + 7| in builtins.seq intMin (builtins.seq b (intMin / b)) + | ^ + 8| + + … while calling the 'seq' builtin + at /pwd/lang/eval-fail-overflowing-div.nix:7:25: + 6| b = -1; + 7| in builtins.seq intMin (builtins.seq b (intMin / b)) + | ^ + 8| + + … while calling the 'div' builtin + at /pwd/lang/eval-fail-overflowing-div.nix:7:48: + 6| b = -1; + 7| in builtins.seq intMin (builtins.seq b (intMin / b)) + | ^ + 8| + + error: integer overflow in dividing -9223372036854775808 / -1 diff --git a/tests/functional/lang/eval-fail-overflowing-div.nix b/tests/functional/lang/eval-fail-overflowing-div.nix new file mode 100644 index 000000000..44fbe9d7e --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-div.nix @@ -0,0 +1,7 @@ +let + # lol, this has to be written as an expression like this because negative + # numbers use unary negation rather than parsing directly, and 2**63 is out + # of range + intMin = -9223372036854775807 - 1; + b = -1; +in builtins.seq intMin (builtins.seq b (intMin / b)) diff --git a/tests/functional/lang/eval-fail-overflowing-mul.err.exp b/tests/functional/lang/eval-fail-overflowing-mul.err.exp new file mode 100644 index 000000000..f42b39d4d --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-mul.err.exp @@ -0,0 +1,16 @@ +error: + … while calling the 'mul' builtin + at /pwd/lang/eval-fail-overflowing-mul.nix:3:10: + 2| a = 4294967297; + 3| in a * a * a + | ^ + 4| + + … while calling the 'mul' builtin + at /pwd/lang/eval-fail-overflowing-mul.nix:3:6: + 2| a = 4294967297; + 3| in a * a * a + | ^ + 4| + + error: integer overflow in multiplying 4294967297 * 4294967297 diff --git a/tests/functional/lang/eval-fail-overflowing-mul.nix b/tests/functional/lang/eval-fail-overflowing-mul.nix new file mode 100644 index 000000000..6081d9c7b --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-mul.nix @@ -0,0 +1,3 @@ +let + a = 4294967297; +in a * a * a diff --git a/tests/functional/lang/eval-fail-overflowing-sub.err.exp b/tests/functional/lang/eval-fail-overflowing-sub.err.exp new file mode 100644 index 000000000..66a3a03f8 --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-sub.err.exp @@ -0,0 +1,9 @@ +error: + … while calling the 'sub' builtin + at /pwd/lang/eval-fail-overflowing-sub.nix:4:6: + 3| b = 2; + 4| in a - b + | ^ + 5| + + error: integer overflow in subtracting -9223372036854775807 - 2 diff --git a/tests/functional/lang/eval-fail-overflowing-sub.nix b/tests/functional/lang/eval-fail-overflowing-sub.nix new file mode 100644 index 000000000..229b8c6d2 --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-sub.nix @@ -0,0 +1,4 @@ +let + a = -9223372036854775807; + b = 2; +in a - b diff --git a/tests/unit/libexpr-support/tests/libexpr.hh b/tests/unit/libexpr-support/tests/libexpr.hh index 54a4a0e9e..045607e87 100644 --- a/tests/unit/libexpr-support/tests/libexpr.hh +++ b/tests/unit/libexpr-support/tests/libexpr.hh @@ -85,7 +85,7 @@ namespace nix { if (arg.type() != nInt) { return false; } - return arg.integer() == v; + return arg.integer().value == v; } MATCHER_P(IsFloatEq, v, fmt("The float is equal to \"%1%\"", v)) { From bf050d9e966534eeb92188da1a93ac459bed0f97 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Fri, 12 Jul 2024 18:20:14 +0200 Subject: [PATCH 101/142] docs: update to define integer overflow Change-Id: Ie8a1b31035f2d27a220e5df2e9e178ec3b39ee68 --- doc/manual/src/language/operators.md | 8 ++++++-- doc/manual/src/language/syntax.md | 7 +++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index e1c020781..2c2dcc276 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -67,8 +67,12 @@ After evaluating *attrset* and *attrpath*, the computational complexity is O(log ## Arithmetic -Numbers are type-compatible: -Pure integer operations will always return integers, whereas any operation involving at least one floating point number return a floating point number. +Numbers will retain their type unless mixed with other numeric types: +Pure integer operations will always return integers, whereas any operation involving at least one floating point number returns a floating point number. + +Evaluation of the following numeric operations throws an evaluation error: +- Division by zero +- Integer overflow, that is, any operation yielding a result outside of the representable range of [Nix language integers](./syntax.md#number-literal) See also [Comparison] and [Equality]. diff --git a/doc/manual/src/language/syntax.md b/doc/manual/src/language/syntax.md index 6108bacd6..f05935502 100644 --- a/doc/manual/src/language/syntax.md +++ b/doc/manual/src/language/syntax.md @@ -183,6 +183,13 @@ This section covers syntax and semantics of the Nix language. Numbers, which can be *integers* (like `123`) or *floating point* (like `123.43` or `.27e13`). + Integers in the Nix language are 64-bit [two's complement] signed integers, with a range of -9223372036854775808 to 9223372036854775807, inclusive. + + [two's complement]: https://en.wikipedia.org/wiki/Two%27s_complement + + Note that negative numeric literals are actually parsed as unary negation of positive numeric literals. + This means that the minimum integer `-9223372036854775808` cannot be written as-is as a literal, since the positive number `9223372036854775808` is one past the maximum range. + See [arithmetic] and [comparison] operators for semantics. [arithmetic]: ./operators.md#arithmetic From 5878b1475f1c50b42672ed85e7259e7d328920e9 Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Thu, 25 Jul 2024 23:06:53 +0200 Subject: [PATCH 102/142] doc: release notes for banning integer overflow Change-Id: Ib75ab5b8b4d879035d7ee7678f9cd0c491a39c0a --- doc/manual/rl-next/ban-integer-overflow.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 doc/manual/rl-next/ban-integer-overflow.md diff --git a/doc/manual/rl-next/ban-integer-overflow.md b/doc/manual/rl-next/ban-integer-overflow.md new file mode 100644 index 000000000..49ecfe4c6 --- /dev/null +++ b/doc/manual/rl-next/ban-integer-overflow.md @@ -0,0 +1,21 @@ +--- +synopsis: Define integer overflow in the Nix language as an error +issues: [10968] +prs: [11188] +--- + +Previously, integer overflow in the Nix language invoked C++ level signed overflow, which was undefined behaviour, but *probably* manifested as wrapping around on overflow. + +Since prior to the public release of Lix, Lix had C++ signed overflow defined to crash the process and nobody noticed this having accidentally removed overflow from the Nix language for three months until it was caught by fiddling around. +Given the significant body of actual Nix code that has been evaluated by Lix in that time, it does not appear that nixpkgs or much of importance depends on integer overflow, so it is safe to turn into an error. + +Some other overflows were fixed: +- `builtins.fromJSON` of values greater than the maximum representable value in a signed 64-bit integer will generate an error. +- `nixConfig` in flakes will no longer accept negative values for configuration options. + +Integer overflow now looks like the following: + +``` +» nix eval --expr '9223372036854775807 + 1' +error: integer overflow in adding 9223372036854775807 + 1 +``` From db5bacb63772138036bdab4cb89a83bb52bbbcfa Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Wed, 31 Jul 2024 21:41:26 +0200 Subject: [PATCH 103/142] reword documentation on `nix-path` config option (#7772) * docs: unify documentation on search paths - put all the information on search path semantics into `builtins.findFile` - put all the information on determining the value of `builtins.nixPath` into the `nix-path` setting maybe `builtins.nixPath` is a better place for this, but those bits can still be moved around now that it's all next to each other. - link to the syntax page for lookup paths from all places that are concerned with it - add or clarify examples - add a test verifying a claim from documentation --- doc/manual/src/command-ref/env-common.md | 26 ++-- doc/manual/src/command-ref/opt-common.md | 9 +- src/libcmd/common-eval-args.cc | 70 +---------- src/libexpr/eval-settings.hh | 21 ++-- src/libexpr/primops.cc | 148 ++++++++++++++++++----- tests/functional/nix_path.sh | 16 +-- 6 files changed, 165 insertions(+), 125 deletions(-) diff --git a/doc/manual/src/command-ref/env-common.md b/doc/manual/src/command-ref/env-common.md index d3f5f9c14..0b5017882 100644 --- a/doc/manual/src/command-ref/env-common.md +++ b/doc/manual/src/command-ref/env-common.md @@ -9,22 +9,26 @@ Most Nix commands interpret the following environment variables: - [`NIX_PATH`](#env-NIX_PATH) - A colon-separated list of directories used to look up the location of Nix - expressions using [paths](@docroot@/language/types.md#type-path) - enclosed in angle brackets (i.e., ``), - e.g. `/home/eelco/Dev:/etc/nixos`. It can be extended using the - [`-I` option](@docroot@/command-ref/opt-common.md#opt-I). + A colon-separated list of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). - If `NIX_PATH` is not set at all, Nix will fall back to the following list in [impure](@docroot@/command-ref/conf-file.md#conf-pure-eval) and [unrestricted](@docroot@/command-ref/conf-file.md#conf-restrict-eval) evaluation mode: + This environment variable overrides the value of the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path). - 1. `$HOME/.nix-defexpr/channels` - 2. `nixpkgs=/nix/var/nix/profiles/per-user/root/channels/nixpkgs` - 3. `/nix/var/nix/profiles/per-user/root/channels` + It can be extended using the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I). + + > **Example** + > + > ```bash + > $ export NIX_PATH=`/home/eelco/Dev:nixos-config=/etc/nixos + > ``` If `NIX_PATH` is set to an empty string, resolving search paths will always fail. - For example, attempting to use `` will produce: - error: file 'nixpkgs' was not found in the Nix search path + > **Example** + > + > ```bash + > $ NIX_PATH= nix-instantiate --eval '' + > error: file 'nixpkgs' was not found in the Nix search path (add it using $NIX_PATH or -I) + > ``` - [`NIX_IGNORE_SYMLINK_STORE`](#env-NIX_IGNORE_SYMLINK_STORE) diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index a42909e2d..69a700207 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -37,7 +37,7 @@ Most Nix commands accept the following command-line options: Print even more informational messages. - `4` “Debug” - + Print debug information. - `5` “Vomit” @@ -187,11 +187,12 @@ Most Nix commands accept the following command-line options: For `nix-shell`, this option is commonly used to give you a shell in which you can build the packages returned by the expression. If you want to get a shell which contain the *built* packages ready for use, give your expression to the `nix-shell --packages ` convenience flag instead. -- [`-I`](#opt-I) *path* +- [`-I` / `--include`](#opt-I) *path* - Add an entry to the [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path). + Add an entry to the list of search paths used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). This option may be given multiple times. - Paths added through `-I` take precedence over [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH). + + Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH). - [`--option`](#opt-option) *name* *value* diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index decadd751..fcef92487 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -91,75 +91,11 @@ MixEvalArgs::MixEvalArgs() .longName = "include", .shortName = 'I', .description = R"( - Add *path* to the Nix search path. The Nix search path is - initialized from the colon-separated [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) environment - variable, and is used to look up the location of Nix expressions using [paths](@docroot@/language/types.md#type-path) enclosed in angle - brackets (i.e., ``). + Add *path* to search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md) - For instance, passing + This option may be given multiple times. - ``` - -I /home/eelco/Dev - -I /etc/nixos - ``` - - will cause Nix to look for paths relative to `/home/eelco/Dev` and - `/etc/nixos`, in that order. This is equivalent to setting the - `NIX_PATH` environment variable to - - ``` - /home/eelco/Dev:/etc/nixos - ``` - - It is also possible to match paths against a prefix. For example, - passing - - ``` - -I nixpkgs=/home/eelco/Dev/nixpkgs-branch - -I /etc/nixos - ``` - - will cause Nix to search for `` in - `/home/eelco/Dev/nixpkgs-branch/path` and `/etc/nixos/nixpkgs/path`. - - If a path in the Nix search path starts with `http://` or `https://`, - it is interpreted as the URL of a tarball that will be downloaded and - unpacked to a temporary location. The tarball must consist of a single - top-level directory. For example, passing - - ``` - -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/master.tar.gz - ``` - - tells Nix to download and use the current contents of the `master` - branch in the `nixpkgs` repository. - - The URLs of the tarballs from the official `nixos.org` channels - (see [the manual page for `nix-channel`](../nix-channel.md)) can be - abbreviated as `channel:`. For instance, the - following two flags are equivalent: - - ``` - -I nixpkgs=channel:nixos-21.05 - -I nixpkgs=https://nixos.org/channels/nixos-21.05/nixexprs.tar.xz - ``` - - You can also fetch source trees using [flake URLs](./nix3-flake.md#url-like-syntax) and add them to the - search path. For instance, - - ``` - -I nixpkgs=flake:nixpkgs - ``` - - specifies that the prefix `nixpkgs` shall refer to the source tree - downloaded from the `nixpkgs` entry in the flake registry. Similarly, - - ``` - -I nixpkgs=flake:github:NixOS/nixpkgs/nixos-22.05 - ``` - - makes `` refer to a particular branch of the - `NixOS/nixpkgs` repository on GitHub. + Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH). )", .category = category, .labels = {"path"}, diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index 30a8c5c58..0cfc14c1b 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -79,19 +79,24 @@ struct EvalSettings : Config This setting determines the value of [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath) and can be used with [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile). - The default value is + - The configuration setting is overridden by the [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) + environment variable. + - `NIX_PATH` is overridden by [specifying the setting as the command line flag](@docroot@/command-ref/conf-file.md#command-line-flags) `--nix-path`. + - Any current value is extended by the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I) or `--extra-nix-path`. - ``` - $HOME/.nix-defexpr/channels - nixpkgs=$NIX_STATE_DIR/profiles/per-user/root/channels/nixpkgs - $NIX_STATE_DIR/profiles/per-user/root/channels - ``` + If the respective paths are accessible, the default values are: - It can be overridden with the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH) or the [`-I` command line option](@docroot@/command-ref/opt-common.md#opt-I). + - `$HOME/.nix-defexpr/channels` + - `nixpkgs=$NIX_STATE_DIR/profiles/per-user/root/channels/nixpkgs` + - `$NIX_STATE_DIR/profiles/per-user/root/channels` + + See [`NIX_STATE_DIR`](@docroot@/command-ref/env-common.md#env-NIX_STATE_DIR) for details. > **Note** > - > If [pure evaluation](#conf-pure-eval) is enabled, `nixPath` evaluates to the empty list `[ ]`. + > If [restricted evaluation](@docroot@/command-ref/conf-file.md#conf-restrict-eval) is enabled, the default value is empty. + > + > If [pure evaluation](#conf-pure-eval) is enabled, `builtins.nixPath` *always* evaluates to the empty list `[ ]`. )", {}, false}; Setting currentSystem{ diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 0b3b19b57..7ceb84f0e 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1843,34 +1843,6 @@ static RegisterPrimOp primop_findFile(PrimOp { .doc = R"( Find *lookup-path* in *search-path*. - A search path is represented list of [attribute sets](./types.md#attribute-set) with two attributes: - - `prefix` is a relative path. - - `path` denotes a file system location - The exact syntax depends on the command line interface. - - Examples of search path attribute sets: - - - ``` - { - prefix = "nixos-config"; - path = "/etc/nixos/configuration.nix"; - } - ``` - - - ``` - { - prefix = ""; - path = "/nix/var/nix/profiles/per-user/root/channels"; - } - ``` - - The lookup algorithm checks each entry until a match is found, returning a [path value](@docroot@/language/types.md#type-path) of the match: - - - If *lookup-path* matches `prefix`, then the remainder of *lookup-path* (the "suffix") is searched for within the directory denoted by `path`. - Note that the `path` may need to be downloaded at this point to look inside. - - If the suffix is found inside that directory, then the entry is a match. - The combined absolute path of the directory (now downloaded if need be) and the suffix is returned. - [Lookup path](@docroot@/language/constructs/lookup-path.md) expressions are [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.nixPath`](#builtins-nixPath): ```nix @@ -1882,6 +1854,119 @@ static RegisterPrimOp primop_findFile(PrimOp { ```nix builtins.findFile builtins.nixPath "nixpkgs" ``` + + A search path is represented as a list of [attribute sets](./types.md#attribute-set) with two attributes: + - `prefix` is a relative path. + - `path` denotes a file system location + + Examples of search path attribute sets: + + - ``` + { + prefix = ""; + path = "/nix/var/nix/profiles/per-user/root/channels"; + } + ``` + - ``` + { + prefix = "nixos-config"; + path = "/etc/nixos/configuration.nix"; + } + ``` + - ``` + { + prefix = "nixpkgs"; + path = "https://github.com/NixOS/nixpkgs/tarballs/master"; + } + ``` + - ``` + { + prefix = "nixpkgs"; + path = "channel:nixpkgs-unstable"; + } + ``` + - ``` + { + prefix = "flake-compat"; + path = "flake:github:edolstra/flake-compat"; + } + ``` + + The lookup algorithm checks each entry until a match is found, returning a [path value](@docroot@/language/types.md#type-path) of the match: + + - If a prefix of `lookup-path` matches `prefix`, then the remainder of *lookup-path* (the "suffix") is searched for within the directory denoted by `path`. + The contents of `path` may need to be downloaded at this point to look inside. + + - If the suffix is found inside that directory, then the entry is a match. + The combined absolute path of the directory (now downloaded if need be) and the suffix is returned. + + > **Example** + > + > A *search-path* value + > + > ``` + > [ + > { + > prefix = ""; + > path = "/home/eelco/Dev"; + > } + > { + > prefix = "nixos-config"; + > path = "/etc/nixos"; + > } + > ] + > ``` + > + > and a *lookup-path* value `"nixos-config"` will cause Nix to try `/home/eelco/Dev/nixos-config` and `/etc/nixos` in that order and return the first path that exists. + + If `path` starts with `http://` or `https://`, it is interpreted as the URL of a tarball that will be downloaded and unpacked to a temporary location. + The tarball must consist of a single top-level directory. + + The URLs of the tarballs from the official `nixos.org` channels can be abbreviated as `channel:`. + See [documentation on `nix-channel`](@docroot@/command-ref/nix-channel.md) for details about channels. + + > **Example** + > + > These two search path entries are equivalent: + > + > - ``` + > { + > prefix = "nixpkgs"; + > path = "channel:nixpkgs-unstable"; + > } + > ``` + > - ``` + > { + > prefix = "nixpkgs"; + > path = "https://nixos.org/channels/nixos-unstable/nixexprs.tar.xz"; + > } + > ``` + + Search paths can also point to source trees using [flake URLs](@docroot@/command-ref/new-cli/nix3-flake.md#url-like-syntax). + + + > **Example** + > + > The search path entry + > + > ``` + > { + > prefix = "nixpkgs"; + > path = "flake:nixpkgs"; + > } + > ``` + > specifies that the prefix `nixpkgs` shall refer to the source tree downloaded from the `nixpkgs` entry in the flake registry. + > + > Similarly + > + > ``` + > { + > prefix = "nixpkgs"; + > path = "flake:github:nixos/nixpkgs/nixos-22.05"; + > } + > ``` + > + > makes `` refer to a particular branch of the `NixOS/nixpkgs` repository on GitHub. )", .fun = prim_findFile, }); @@ -4731,6 +4816,13 @@ void EvalState::createBaseEnv() .doc = R"( The value of the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path): a list of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). + > **Example** + > + > ```bash + > $ NIX_PATH= nix-instantiate --eval --expr "builtins.nixPath" -I foo=bar --no-pure-eval + > [ { path = "bar"; prefix = "foo"; } ] + > ``` + Lookup path expressions are [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.findFile`](./builtins.html#builtins-findFile): diff --git a/tests/functional/nix_path.sh b/tests/functional/nix_path.sh index 7e6a0458d..90cba1f0c 100755 --- a/tests/functional/nix_path.sh +++ b/tests/functional/nix_path.sh @@ -22,13 +22,13 @@ nix-instantiate --eval -E '' --restrict-eval # # | precedence | hard-coded | nix-path in file | extra-nix-path in file | nix-path in env | extra-nix-path in env | NIX_PATH | nix-path | extra-nix-path | -I | # |------------------------|------------|------------------|------------------------|-----------------|-----------------------|-----------|-----------|-----------------|-----------------| -# | hard-coded | x | ^override | ^append | ^override | ^append | ^override | ^override | ^append | ^append | -# | nix-path in file | | last wins | ^append | ^override | ^append | ^override | ^override | ^append | ^append | -# | extra-nix-path in file | | | append in order | ^override | ^append | ^override | ^override | ^append | ^append | -# | nix-path in env | | | | last wins | ^append | ^override | ^override | ^append | ^append | -# | extra-nix-path in env | | | | | append in order | ^override | ^override | ^append | ^append | -# | NIX_PATH | | | | | | x | ^override | ^append | ^append | -# | nix-path | | | | | | | last wins | ^append | ^append | +# | hard-coded | x | ^override | ^append | ^override | ^append | ^override | ^override | ^append | ^prepend | +# | nix-path in file | | last wins | ^append | ^override | ^append | ^override | ^override | ^append | ^prepend | +# | extra-nix-path in file | | | append in order | ^override | ^append | ^override | ^override | ^append | ^prepend | +# | nix-path in env | | | | last wins | ^append | ^override | ^override | ^append | ^prepend | +# | extra-nix-path in env | | | | | append in order | ^override | ^override | ^append | ^prepend | +# | NIX_PATH | | | | | | x | ^override | ^append | ^prepend | +# | nix-path | | | | | | | last wins | ^append | ^prepend | # | extra-nix-path | | | | | | | | append in order | append in order | # | -I | | | | | | | | | append in order | @@ -59,6 +59,8 @@ echo "nix-path = test=$TEST_ROOT/from-nix-path-file" >> "$test_nix_conf" # -I extends NIX_PATH [[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate -I test=$TEST_ROOT/from-I --find-file test/only-from-I.nix) = $TEST_ROOT/from-I/only-from-I.nix ]] +# -I takes precedence over NIX_PATH +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate -I test=$TEST_ROOT/from-I --find-file test) = $TEST_ROOT/from-I ]] # if -I does not have the desired entry, the value from NIX_PATH is used [[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate -I test=$TEST_ROOT/from-I --find-file test/only-from-NIX_PATH.nix) = $TEST_ROOT/from-NIX_PATH/only-from-NIX_PATH.nix ]] From c952d933e574dbc17304b18b33a9afeed7ebd966 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 31 Jul 2024 21:57:31 +0200 Subject: [PATCH 104/142] release notes: 2.24.0 --- .../10564-attrcursor-remove-forceerrors.md | 9 - ...03-run-the-flake-regressions-test-suite.md | 8 - ...unit-prefixes-in-configuration-settings.md | 10 - ...ild-show-all-fod-errors-with-keep-going.md | 10 - doc/manual/rl-next/10855-meson.md | 31 -- .../11086-eval-cache-fix-cache-regressions.md | 14 - .../rl-next/9063-introduce-libnixflake.md | 12 - doc/manual/rl-next/drop-vendored-toml11.md | 8 - doc/manual/rl-next/harden-user-sandboxing.md | 12 - .../rl-next/nix-shell-looks-for-shell-nix.md | 30 -- doc/manual/rl-next/pipe-operators.md | 28 -- .../rl-next/repl-doc-renders-doc-comments.md | 55 ---- doc/manual/rl-next/shebang-relative.md | 64 ---- doc/manual/rl-next/tarball-fixes.md | 9 - doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/release-notes/rl-2.24.md | 300 ++++++++++++++++++ 16 files changed, 301 insertions(+), 300 deletions(-) delete mode 100644 doc/manual/rl-next/10564-attrcursor-remove-forceerrors.md delete mode 100644 doc/manual/rl-next/10603-run-the-flake-regressions-test-suite.md delete mode 100644 doc/manual/rl-next/10668-support-unit-prefixes-in-configuration-settings.md delete mode 100644 doc/manual/rl-next/10734-nix3-build-show-all-fod-errors-with-keep-going.md delete mode 100644 doc/manual/rl-next/10855-meson.md delete mode 100644 doc/manual/rl-next/11086-eval-cache-fix-cache-regressions.md delete mode 100644 doc/manual/rl-next/9063-introduce-libnixflake.md delete mode 100644 doc/manual/rl-next/drop-vendored-toml11.md delete mode 100644 doc/manual/rl-next/harden-user-sandboxing.md delete mode 100644 doc/manual/rl-next/nix-shell-looks-for-shell-nix.md delete mode 100644 doc/manual/rl-next/pipe-operators.md delete mode 100644 doc/manual/rl-next/repl-doc-renders-doc-comments.md delete mode 100644 doc/manual/rl-next/shebang-relative.md delete mode 100644 doc/manual/rl-next/tarball-fixes.md create mode 100644 doc/manual/src/release-notes/rl-2.24.md diff --git a/doc/manual/rl-next/10564-attrcursor-remove-forceerrors.md b/doc/manual/rl-next/10564-attrcursor-remove-forceerrors.md deleted file mode 100644 index 864a55b51..000000000 --- a/doc/manual/rl-next/10564-attrcursor-remove-forceerrors.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -synopsis: "Solve `cached failure of attribute X`" -prs: 10564 -issues: 10513 9165 ---- - -This eliminates all "cached failure of attribute X" messages by forcing evaluation of the original value when needed to show the exception to the user. This enhancement improves error reporting by providing the underlying message and stack trace. - -Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) diff --git a/doc/manual/rl-next/10603-run-the-flake-regressions-test-suite.md b/doc/manual/rl-next/10603-run-the-flake-regressions-test-suite.md deleted file mode 100644 index 42864323c..000000000 --- a/doc/manual/rl-next/10603-run-the-flake-regressions-test-suite.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -synopsis: "Run the flake regressions test suite" -prs: 10603 ---- - -This update introduces a GitHub action to run a subset of the [flake regressions test suite](https://github.com/NixOS/flake-regressions), which includes 259 flakes with their expected evaluation results. Currently, the action runs the first 25 flakes due to the full test suite's extensive runtime. A manually triggered action may be implemented later to run the entire test suite. - -Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) diff --git a/doc/manual/rl-next/10668-support-unit-prefixes-in-configuration-settings.md b/doc/manual/rl-next/10668-support-unit-prefixes-in-configuration-settings.md deleted file mode 100644 index 2caca9a81..000000000 --- a/doc/manual/rl-next/10668-support-unit-prefixes-in-configuration-settings.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -synopsis: "Support unit prefixes in configuration settings" -prs: 10668 ---- - -Configuration settings in Nix now support unit prefixes, allowing for more intuitive and readable configurations. For example, you can now specify [`--min-free 1G`](@docroot@/command-ref/opt-common.md#opt-min-free) to set the minimum free space to 1 gigabyte. - -This enhancement was extracted from [#7851](https://github.com/NixOS/nix/pull/7851) and is also useful for PR [#10661](https://github.com/NixOS/nix/pull/10661). - -Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) diff --git a/doc/manual/rl-next/10734-nix3-build-show-all-fod-errors-with-keep-going.md b/doc/manual/rl-next/10734-nix3-build-show-all-fod-errors-with-keep-going.md deleted file mode 100644 index 1d623e952..000000000 --- a/doc/manual/rl-next/10734-nix3-build-show-all-fod-errors-with-keep-going.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -synopsis: "nix3-build: show all FOD errors with `--keep-going`" -prs: 10734 ---- - -The [`nix build`](@docroot@/command-ref/new-cli/nix3-build.md) command has been updated to improve the behavior of the [`--keep-going`] flag. Now, when `--keep-going` is used, all hash-mismatch errors of failing fixed-output derivations (FODs) are displayed, similar to the behavior for other build failures. This enhancement ensures that all relevant build errors are shown, making it easier for users to update multiple derivations at once or to diagnose and fix issues. - -Author: [**Jörg Thalheim (@Mic92)**](https://github.com/Mic92), [**Maximilian Bosch (@Ma27)**](https://github.com/Ma27) - -[`--keep-going`](@docroot@/command-ref/opt-common.md#opt-keep-going) diff --git a/doc/manual/rl-next/10855-meson.md b/doc/manual/rl-next/10855-meson.md deleted file mode 100644 index 0ab71390f..000000000 --- a/doc/manual/rl-next/10855-meson.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -synopsis: "Build with Meson" -prs: -- 10378 -- 10855 -- 10904 -- 10908 -- 10914 -- 10933 -- 10936 -- 10954 -- 10955 -- 10967 -- 10963 -- 10973 -- 11034 -- 11054 -- 11055 -- 11064 -- 11060 -- 11155 -issues: -- 2503 ---- - -These changes aim to replace the use of autotools and make with Meson for building various components of Nix. Additionally, each library is built in its own derivation, leveraging Meson's "subprojects" feature to allow a single development shell for building all libraries while also supporting separate builds. This approach aims to improve productivity and build modularity, compared to both make and a monolithic Meson-based derivation. - -Special thanks to everyone who has contributed to the Meson port, particularly [**@p01arst0rm**](https://github.com/p01arst0rm) and [**@Qyriad**](https://github.com/Qyriad). - -Authors: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314), [**Tom Bereknyei**](https://github.com/tomberek), [**Théophane Hufschmitt (@thufschmitt)**](https://github.com/thufschmitt), [**Valentin Gagarin (@fricklerhandwerk)**](https://github.com/fricklerhandwerk), [**Robert Hensing (@roberth)**](https://github.com/roberth) -Co-authors: [**@p01arst0rm**](https://github.com/p01arst0rm), [**@Qyriad**](https://github.com/Qyriad) diff --git a/doc/manual/rl-next/11086-eval-cache-fix-cache-regressions.md b/doc/manual/rl-next/11086-eval-cache-fix-cache-regressions.md deleted file mode 100644 index 8a348a9ad..000000000 --- a/doc/manual/rl-next/11086-eval-cache-fix-cache-regressions.md +++ /dev/null @@ -1,14 +0,0 @@ ---- -synopsis: "Eval cache: fix cache regressions" -prs: 11086 -issues: 10570 ---- - -This update addresses two bugs in the evaluation cache system: - -1. Regression in #10570: The evaluation cache was not being persisted in `nix develop` because `evalCaches` retained references to the caches and was never freed. -2. Nix could sometimes try to commit the evaluation cache SQLite transaction without there being an active transaction, resulting in non-error errors being printed. - -These bug fixes ensure that the evaluation cache is correctly managed and errors are appropriately handled. - -Author: [**Lexi Mattick (@kognise)**](https://github.com/kognise) diff --git a/doc/manual/rl-next/9063-introduce-libnixflake.md b/doc/manual/rl-next/9063-introduce-libnixflake.md deleted file mode 100644 index fd3645446..000000000 --- a/doc/manual/rl-next/9063-introduce-libnixflake.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -synopsis: "Introduce `libnixflake`" -prs: 9063 ---- - -A new library, `libnixflake`, has been introduced to better separate the Flakes layer within Nix. This change refactors the codebase to encapsulate Flakes-specific functionality within its own library. - -See the commits in the pull request for detailed changes, with the only significant code modifications happening in the initial commit. - -This change was alluded to in [RFC 134](https://github.com/nixos/rfcs/blob/master/rfcs/0134-nix-store-layer.md) and is a step towards a more modular and maintainable codebase. - -Author: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314) diff --git a/doc/manual/rl-next/drop-vendored-toml11.md b/doc/manual/rl-next/drop-vendored-toml11.md deleted file mode 100644 index 8dd786c44..000000000 --- a/doc/manual/rl-next/drop-vendored-toml11.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -synopsis: Stop vendoring toml11 ---- - -We don't apply any patches to it, and vendoring it locks users into -bugs (it hasn't been updated since its introduction in late 2021). - -Author: [**Winter (@winterqt)**](https://github.com/winterqt) diff --git a/doc/manual/rl-next/harden-user-sandboxing.md b/doc/manual/rl-next/harden-user-sandboxing.md deleted file mode 100644 index ff81c9cb1..000000000 --- a/doc/manual/rl-next/harden-user-sandboxing.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -synopsis: Harden the user sandboxing -significance: significant -issues: ---- - -The build directory has been hardened against interference with the outside world by nesting it inside another directory owned by (and only readable by) the daemon user. - -This is a low severity security fix, [CVE-2024-38531](https://www.cve.org/CVERecord?id=CVE-2024-38531), that was handled through the GitHub Security Advisories interface, and hence was merged directly in commit [2dd7f8f42](https://github.com/NixOS/nix/commit/2dd7f8f42da374d9fee4d424c1c6f82bcb36b393) instead of a PR. - -Credit: [**@alois31**](https://github.com/alois31), [**Linus Heckemann (@lheckemann)**](https://github.com/lheckemann) -Co-authors: [**@edolstra**](https://github.com/edolstra) diff --git a/doc/manual/rl-next/nix-shell-looks-for-shell-nix.md b/doc/manual/rl-next/nix-shell-looks-for-shell-nix.md deleted file mode 100644 index b9e4b3fb3..000000000 --- a/doc/manual/rl-next/nix-shell-looks-for-shell-nix.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -synopsis: "`nix-shell ` looks for `shell.nix`" -significance: significant -issues: -- 496 -- 2279 -- 4529 -- 5431 -- 11053 -prs: -- 11057 ---- - -`nix-shell $x` now looks for `$x/shell.nix` when `$x` resolves to a directory. - -Although this might be seen as a breaking change, its primarily interactive usage makes it a minor issue. -This adjustment addresses a commonly reported problem. - -This also applies to `nix-shell` shebang scripts. Consider the following example: - -```shell -#!/usr/bin/env nix-shell -#!nix-shell -i bash -``` - -This will now load `shell.nix` from the script's directory, if it exists; `default.nix` otherwise. - -The old behavior can be opted into by setting the option [`nix-shell-always-looks-for-shell-nix`](@docroot@/command-ref/conf-file.md#conf-nix-shell-always-looks-for-shell-nix) to `false`. - -Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) diff --git a/doc/manual/rl-next/pipe-operators.md b/doc/manual/rl-next/pipe-operators.md deleted file mode 100644 index b4cbe30e3..000000000 --- a/doc/manual/rl-next/pipe-operators.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -synopsis: "Add `pipe-operators` experimental feature" -prs: -- 11131 ---- - -This is a draft implementation of [RFC 0148](https://github.com/NixOS/rfcs/pull/148). - -The `pipe-operators` experimental feature adds [`<|` and `|>` operators][pipe operators] to the Nix language. -*a* `|>` *b* is equivalent to the function application *b* *a*, and -*a* `<|` *b* is equivalent to the function application *a* *b*. - -For example: - -``` -nix-repl> 1 |> builtins.add 2 |> builtins.mul 3 -9 - -nix-repl> builtins.add 1 <| builtins.mul 2 <| 3 -7 -``` - -`<|` and `|>` are right and left associative, respectively, and have lower precedence than any other operator. -These properties may change in future releases. - -See [the RFC](https://github.com/NixOS/rfcs/pull/148) for more examples and rationale. - -[pipe operators]: @docroot@/language/operators.md#pipe-operators diff --git a/doc/manual/rl-next/repl-doc-renders-doc-comments.md b/doc/manual/rl-next/repl-doc-renders-doc-comments.md deleted file mode 100644 index fa241ebc1..000000000 --- a/doc/manual/rl-next/repl-doc-renders-doc-comments.md +++ /dev/null @@ -1,55 +0,0 @@ ---- -synopsis: "`nix-repl`'s `:doc` shows documentation comments" -significance: significant -issues: -- 3904 -- 10771 -prs: -- 1652 -- 9054 -- 11072 ---- - -`nix repl` has a `:doc` command that previously only rendered documentation for internally defined functions. -This feature has been extended to also render function documentation comments, in accordance with [RFC 145]. - -Example: - -``` -nix-repl> :doc lib.toFunction -Function toFunction - … defined at /home/user/h/nixpkgs/lib/trivial.nix:1072:5 - - Turns any non-callable values into constant functions. Returns - callable values as is. - -Inputs - - v - - : Any value - -Examples - - :::{.example} - -## lib.trivial.toFunction usage example - - | nix-repl> lib.toFunction 1 2 - | 1 - | - | nix-repl> lib.toFunction (x: x + 1) 2 - | 3 - - ::: -``` - -Known limitations: -- It does not render documentation for "formals", such as `{ /** the value to return */ x, ... }: x`. -- Some extensions to markdown are not yet supported, as you can see in the example above. - -We'd like to acknowledge [Yingchi Long (@inclyc)](https://github.com/inclyc) for proposing a proof of concept for this functionality in [#9054](https://github.com/NixOS/nix/pull/9054), as well as [@sternenseemann](https://github.com/sternenseemann) and [Johannes Kirschbauer (@hsjobeki)](https://github.com/hsjobeki) for their contributions, proposals, and their work on [RFC 145]. - -Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) - -[RFC 145]: https://github.com/NixOS/rfcs/pull/145 diff --git a/doc/manual/rl-next/shebang-relative.md b/doc/manual/rl-next/shebang-relative.md deleted file mode 100644 index dd96bf203..000000000 --- a/doc/manual/rl-next/shebang-relative.md +++ /dev/null @@ -1,64 +0,0 @@ ---- -synopsis: "`nix-shell` shebang uses relative path" -prs: -- 5088 -- 11058 -issues: -- 4232 ---- - - -Relative [path](@docroot@/language/types.md#type-path) literals in `nix-shell` shebang scripts' options are now resolved relative to the [script's location](@docroot@/glossary.md?highlight=base%20directory#gloss-base-directory). -Previously they were resolved relative to the current working directory. - -For example, consider the following script in `~/myproject/say-hi`: - -```shell -#!/usr/bin/env nix-shell -#!nix-shell --expr 'import ./shell.nix' -#!nix-shell --arg toolset './greeting-tools.nix' -#!nix-shell -i bash -hello -``` - -Older versions of `nix-shell` would resolve `shell.nix` relative to the current working directory; home in this example: - -```console -[hostname:~]$ ./myproject/say-hi -error: - … while calling the 'import' builtin - at «string»:1:2: - 1| (import ./shell.nix) - | ^ - - error: path '/home/user/shell.nix' does not exist -``` - -Since this release, `nix-shell` resolves `shell.nix` relative to the script's location, and `~/myproject/shell.nix` is used. - -```console -$ ./myproject/say-hi -Hello, world! -``` - -**Opt-out** - -This is technically a breaking change, so we have added an option so you can adapt independently of your Nix update. -The old behavior can be opted into by setting the option [`nix-shell-shebang-arguments-relative-to-script`](@docroot@/command-ref/conf-file.md#conf-nix-shell-shebang-arguments-relative-to-script) to `false`. -This option will be removed in a future release. - -**`nix` command shebang** - -The experimental [`nix` command shebang](@docroot@/command-ref/new-cli/nix.md?highlight=shebang#shebang-interpreter) already behaves in this script-relative manner. - -Example: - -```shell -#!/usr/bin/env nix -#!nix develop -#!nix --expr ``import ./shell.nix`` -#!nix -c bash -hello -``` - -Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) diff --git a/doc/manual/rl-next/tarball-fixes.md b/doc/manual/rl-next/tarball-fixes.md deleted file mode 100644 index c938e9db6..000000000 --- a/doc/manual/rl-next/tarball-fixes.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -synopsis: "Improve handling of tarballs that don't consist of a single top-level directory" -prs: -- 11195 ---- - -In previous Nix releases, the tarball fetcher (used by `builtins.fetchTarball`) erroneously merged top-level directories into a single directory, and silently discarded top-level files that are not directories. This is no longer the case. The new behaviour is that *only* if the tarball consists of a single directory, the top-level path component of the files in the tarball is removed (similar to `tar`'s `--strip-components=1`). - -Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 3918faeb2..8739599a0 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -127,6 +127,7 @@ - [Contributing](development/contributing.md) - [Releases](release-notes/index.md) {{#include ./SUMMARY-rl-next.md}} + - [Release 2.24 (2024-07-31)](release-notes/rl-2.24.md) - [Release 2.23 (2024-06-03)](release-notes/rl-2.23.md) - [Release 2.22 (2024-04-23)](release-notes/rl-2.22.md) - [Release 2.21 (2024-03-11)](release-notes/rl-2.21.md) diff --git a/doc/manual/src/release-notes/rl-2.24.md b/doc/manual/src/release-notes/rl-2.24.md new file mode 100644 index 000000000..c68072573 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.24.md @@ -0,0 +1,300 @@ +# Release 2.24.0 (2024-07-31) + +### Significant changes + +- Harden the user sandboxing + + The build directory has been hardened against interference with the outside world by nesting it inside another directory owned by (and only readable by) the daemon user. + + This is a low severity security fix, [CVE-2024-38531](https://www.cve.org/CVERecord?id=CVE-2024-38531), that was handled through the GitHub Security Advisories interface, and hence was merged directly in commit [2dd7f8f42](https://github.com/NixOS/nix/commit/2dd7f8f42da374d9fee4d424c1c6f82bcb36b393) instead of a PR. + + Credit: [**@alois31**](https://github.com/alois31), [**Linus Heckemann (@lheckemann)**](https://github.com/lheckemann) + Co-authors: [**@edolstra**](https://github.com/edolstra) + +- `nix-shell ` looks for `shell.nix` [#496](https://github.com/NixOS/nix/issues/496) [#2279](https://github.com/NixOS/nix/issues/2279) [#4529](https://github.com/NixOS/nix/issues/4529) [#5431](https://github.com/NixOS/nix/issues/5431) [#11053](https://github.com/NixOS/nix/issues/11053) [#11057](https://github.com/NixOS/nix/pull/11057) + + `nix-shell $x` now looks for `$x/shell.nix` when `$x` resolves to a directory. + + Although this might be seen as a breaking change, its primarily interactive usage makes it a minor issue. + This adjustment addresses a commonly reported problem. + + This also applies to `nix-shell` shebang scripts. Consider the following example: + + ```shell + #!/usr/bin/env nix-shell + #!nix-shell -i bash + ``` + + This will now load `shell.nix` from the script's directory, if it exists; `default.nix` otherwise. + + The old behavior can be opted into by setting the option [`nix-shell-always-looks-for-shell-nix`](@docroot@/command-ref/conf-file.md#conf-nix-shell-always-looks-for-shell-nix) to `false`. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + +- `nix-repl`'s `:doc` shows documentation comments [#3904](https://github.com/NixOS/nix/issues/3904) [#10771](https://github.com/NixOS/nix/issues/10771) [#1652](https://github.com/NixOS/nix/pull/1652) [#9054](https://github.com/NixOS/nix/pull/9054) [#11072](https://github.com/NixOS/nix/pull/11072) + + `nix repl` has a `:doc` command that previously only rendered documentation for internally defined functions. + This feature has been extended to also render function documentation comments, in accordance with [RFC 145]. + + Example: + + ``` + nix-repl> :doc lib.toFunction + Function toFunction + … defined at /home/user/h/nixpkgs/lib/trivial.nix:1072:5 + + Turns any non-callable values into constant functions. Returns + callable values as is. + + Inputs + + v + + : Any value + + Examples + + :::{.example} + + ## lib.trivial.toFunction usage example + + | nix-repl> lib.toFunction 1 2 + | 1 + | + | nix-repl> lib.toFunction (x: x + 1) 2 + | 3 + + ::: + ``` + + Known limitations: + - It does not render documentation for "formals", such as `{ /** the value to return */ x, ... }: x`. + - Some extensions to markdown are not yet supported, as you can see in the example above. + + We'd like to acknowledge [Yingchi Long (@inclyc)](https://github.com/inclyc) for proposing a proof of concept for this functionality in [#9054](https://github.com/NixOS/nix/pull/9054), as well as [@sternenseemann](https://github.com/sternenseemann) and [Johannes Kirschbauer (@hsjobeki)](https://github.com/hsjobeki) for their contributions, proposals, and their work on [RFC 145]. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + + [RFC 145]: https://github.com/NixOS/rfcs/pull/145 + +### Other changes + +- Solve `cached failure of attribute X` [#9165](https://github.com/NixOS/nix/issues/9165) [#10513](https://github.com/NixOS/nix/issues/10513) [#10564](https://github.com/NixOS/nix/pull/10564) + + This eliminates all "cached failure of attribute X" messages by forcing evaluation of the original value when needed to show the exception to the user. This enhancement improves error reporting by providing the underlying message and stack trace. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Run the flake regressions test suite [#10603](https://github.com/NixOS/nix/pull/10603) + + This update introduces a GitHub action to run a subset of the [flake regressions test suite](https://github.com/NixOS/flake-regressions), which includes 259 flakes with their expected evaluation results. Currently, the action runs the first 25 flakes due to the full test suite's extensive runtime. A manually triggered action may be implemented later to run the entire test suite. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Support unit prefixes in configuration settings [#10668](https://github.com/NixOS/nix/pull/10668) + + Configuration settings in Nix now support unit prefixes, allowing for more intuitive and readable configurations. For example, you can now specify [`--min-free 1G`](@docroot@/command-ref/opt-common.md#opt-min-free) to set the minimum free space to 1 gigabyte. + + This enhancement was extracted from [#7851](https://github.com/NixOS/nix/pull/7851) and is also useful for PR [#10661](https://github.com/NixOS/nix/pull/10661). + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- nix3-build: show all FOD errors with `--keep-going` [#10734](https://github.com/NixOS/nix/pull/10734) + + The [`nix build`](@docroot@/command-ref/new-cli/nix3-build.md) command has been updated to improve the behavior of the [`--keep-going`] flag. Now, when `--keep-going` is used, all hash-mismatch errors of failing fixed-output derivations (FODs) are displayed, similar to the behavior for other build failures. This enhancement ensures that all relevant build errors are shown, making it easier for users to update multiple derivations at once or to diagnose and fix issues. + + Author: [**Jörg Thalheim (@Mic92)**](https://github.com/Mic92), [**Maximilian Bosch (@Ma27)**](https://github.com/Ma27) + + [`--keep-going`](@docroot@/command-ref/opt-common.md#opt-keep-going) + +- Build with Meson [#2503](https://github.com/NixOS/nix/issues/2503) [#10378](https://github.com/NixOS/nix/pull/10378) [#10855](https://github.com/NixOS/nix/pull/10855) [#10904](https://github.com/NixOS/nix/pull/10904) [#10908](https://github.com/NixOS/nix/pull/10908) [#10914](https://github.com/NixOS/nix/pull/10914) [#10933](https://github.com/NixOS/nix/pull/10933) [#10936](https://github.com/NixOS/nix/pull/10936) [#10954](https://github.com/NixOS/nix/pull/10954) [#10955](https://github.com/NixOS/nix/pull/10955) [#10963](https://github.com/NixOS/nix/pull/10963) [#10967](https://github.com/NixOS/nix/pull/10967) [#10973](https://github.com/NixOS/nix/pull/10973) [#11034](https://github.com/NixOS/nix/pull/11034) [#11054](https://github.com/NixOS/nix/pull/11054) [#11055](https://github.com/NixOS/nix/pull/11055) [#11060](https://github.com/NixOS/nix/pull/11060) [#11064](https://github.com/NixOS/nix/pull/11064) [#11155](https://github.com/NixOS/nix/pull/11155) + + These changes aim to replace the use of autotools and make with Meson for building various components of Nix. Additionally, each library is built in its own derivation, leveraging Meson's "subprojects" feature to allow a single development shell for building all libraries while also supporting separate builds. This approach aims to improve productivity and build modularity, compared to both make and a monolithic Meson-based derivation. + + Special thanks to everyone who has contributed to the Meson port, particularly [**@p01arst0rm**](https://github.com/p01arst0rm) and [**@Qyriad**](https://github.com/Qyriad). + + Authors: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314), [**Tom Bereknyei**](https://github.com/tomberek), [**Théophane Hufschmitt (@thufschmitt)**](https://github.com/thufschmitt), [**Valentin Gagarin (@fricklerhandwerk)**](https://github.com/fricklerhandwerk), [**Robert Hensing (@roberth)**](https://github.com/roberth) + Co-authors: [**@p01arst0rm**](https://github.com/p01arst0rm), [**@Qyriad**](https://github.com/Qyriad) + +- Eval cache: fix cache regressions [#10570](https://github.com/NixOS/nix/issues/10570) [#11086](https://github.com/NixOS/nix/pull/11086) + + This update addresses two bugs in the evaluation cache system: + + 1. Regression in #10570: The evaluation cache was not being persisted in `nix develop` because `evalCaches` retained references to the caches and was never freed. + 2. Nix could sometimes try to commit the evaluation cache SQLite transaction without there being an active transaction, resulting in non-error errors being printed. + + These bug fixes ensure that the evaluation cache is correctly managed and errors are appropriately handled. + + Author: [**Lexi Mattick (@kognise)**](https://github.com/kognise) + +- Introduce `libnixflake` [#9063](https://github.com/NixOS/nix/pull/9063) + + A new library, `libnixflake`, has been introduced to better separate the Flakes layer within Nix. This change refactors the codebase to encapsulate Flakes-specific functionality within its own library. + + See the commits in the pull request for detailed changes, with the only significant code modifications happening in the initial commit. + + This change was alluded to in [RFC 134](https://github.com/nixos/rfcs/blob/master/rfcs/0134-nix-store-layer.md) and is a step towards a more modular and maintainable codebase. + + Author: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314) + +- CL options `--arg-from-file` and `--arg-from-stdin` [#9913](https://github.com/NixOS/nix/pull/9913) + + The `--debugger` now prints source location information, instead of the + pointers of source location information. Before: + + ``` + nix-repl> :bt + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + 0x600001522598 + ``` + + After: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in + ``` + +- Make `nix store gc` use the auto-GC policy [#7851](https://github.com/NixOS/nix/pull/7851) + + + +- Stop vendoring toml11 + + We don't apply any patches to it, and vendoring it locks users into + bugs (it hasn't been updated since its introduction in late 2021). + + Author: [**Winter (@winterqt)**](https://github.com/winterqt) + +- Rename hash format `base32` to `nix32` [#8678](https://github.com/NixOS/nix/pull/8678) + + Hash format `base32` was renamed to `nix32` since it used a special nix-specific character set for + [Base32](https://en.wikipedia.org/wiki/Base32). + + ## Deprecation: Use `nix32` instead of `base32` as `toHashFormat` + + For the builtin `convertHash`, the `toHashFormat` parameter now accepts the same hash formats as the `--to`/`--from` + parameters of the `nix hash conert` command: `"base16"`, `"nix32"`, `"base64"`, and `"sri"`. The former `"base32"` value + remains as a deprecated alias for `"base32"`. Please convert your code from: + + ```nix + builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base32";} + ``` + + to + + ```nix + builtins.convertHash { inherit hash hashAlgo; toHashFormat = "nix32";} + ``` + +- Add `pipe-operators` experimental feature [#11131](https://github.com/NixOS/nix/pull/11131) + + This is a draft implementation of [RFC 0148](https://github.com/NixOS/rfcs/pull/148). + + The `pipe-operators` experimental feature adds [`<|` and `|>` operators][pipe operators] to the Nix language. + *a* `|>` *b* is equivalent to the function application *b* *a*, and + *a* `<|` *b* is equivalent to the function application *a* *b*. + + For example: + + ``` + nix-repl> 1 |> builtins.add 2 |> builtins.mul 3 + 9 + + nix-repl> builtins.add 1 <| builtins.mul 2 <| 3 + 7 + ``` + + `<|` and `|>` are right and left associative, respectively, and have lower precedence than any other operator. + These properties may change in future releases. + + See [the RFC](https://github.com/NixOS/rfcs/pull/148) for more examples and rationale. + + [pipe operators]: @docroot@/language/operators.md#pipe-operators + +- `nix-shell` shebang uses relative path [#4232](https://github.com/NixOS/nix/issues/4232) [#5088](https://github.com/NixOS/nix/pull/5088) [#11058](https://github.com/NixOS/nix/pull/11058) + + + Relative [path](@docroot@/language/types.md#type-path) literals in `nix-shell` shebang scripts' options are now resolved relative to the [script's location](@docroot@/glossary.md?highlight=base%20directory#gloss-base-directory). + Previously they were resolved relative to the current working directory. + + For example, consider the following script in `~/myproject/say-hi`: + + ```shell + #!/usr/bin/env nix-shell + #!nix-shell --expr 'import ./shell.nix' + #!nix-shell --arg toolset './greeting-tools.nix' + #!nix-shell -i bash + hello + ``` + + Older versions of `nix-shell` would resolve `shell.nix` relative to the current working directory; home in this example: + + ```console + [hostname:~]$ ./myproject/say-hi + error: + … while calling the 'import' builtin + at «string»:1:2: + 1| (import ./shell.nix) + | ^ + + error: path '/home/user/shell.nix' does not exist + ``` + + Since this release, `nix-shell` resolves `shell.nix` relative to the script's location, and `~/myproject/shell.nix` is used. + + ```console + $ ./myproject/say-hi + Hello, world! + ``` + + **Opt-out** + + This is technically a breaking change, so we have added an option so you can adapt independently of your Nix update. + The old behavior can be opted into by setting the option [`nix-shell-shebang-arguments-relative-to-script`](@docroot@/command-ref/conf-file.md#conf-nix-shell-shebang-arguments-relative-to-script) to `false`. + This option will be removed in a future release. + + **`nix` command shebang** + + The experimental [`nix` command shebang](@docroot@/command-ref/new-cli/nix.md?highlight=shebang#shebang-interpreter) already behaves in this script-relative manner. + + Example: + + ```shell + #!/usr/bin/env nix + #!nix develop + #!nix --expr ``import ./shell.nix`` + #!nix -c bash + hello + ``` + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + +- Improve handling of tarballs that don't consist of a single top-level directory [#11195](https://github.com/NixOS/nix/pull/11195) + + In previous Nix releases, the tarball fetcher (used by `builtins.fetchTarball`) erroneously merged top-level directories into a single directory, and silently discarded top-level files that are not directories. This is no longer the case. The new behaviour is that *only* if the tarball consists of a single directory, the top-level path component of the files in the tarball is removed (similar to `tar`'s `--strip-components=1`). + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Improve handling of tarballs that don't consist of a single top-level directory [#11195](https://github.com/NixOS/nix/pull/11195) + + In previous Nix releases, the tarball fetcher (used by `builtins.fetchTarball`) erroneously merged top-level directories into a single directory, and silently discarded top-level files that are not directories. This is no longer the case. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Setting to warn about large paths [#10778](https://github.com/NixOS/nix/pull/10778) + + Nix can now warn when evaluation of a Nix expression causes a large + path to be copied to the Nix store. The threshold for this warning can + be configured using the `warn-large-path-threshold` setting, + e.g. `--warn-large-path-threshold 100M`. + + +# Contributors + +Querying GitHub API for ee86e7f361c55c8c7dc2e45c3868802af249aeff, to get handle for MostAwesomeDude@gmail.com From 733c816d3493459e0f7f899111b9ad19fe04147a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 31 Jul 2024 15:04:18 -0500 Subject: [PATCH 105/142] Small windows cross fixes (#11230) --- src/libexpr/eval.cc | 4 +++- src/libstore/gc.cc | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 746ccab2a..de5d85821 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2860,8 +2860,10 @@ void EvalState::printStatistics() topObj["cpuTime"] = cpuTime; #endif topObj["time"] = { +#ifndef _WIN32 // TODO implement {"cpu", cpuTime}, -#ifdef HAVE_BOEHMGC +#endif +#if HAVE_BOEHMGC {GC_is_incremental_mode() ? "gcNonIncremental" : "gc", gcFullOnlyTime}, {GC_is_incremental_mode() ? "gcNonIncrementalFraction" : "gcFraction", gcFullOnlyTime / cpuTime}, #endif diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index c865fddd7..1494712da 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -891,7 +891,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) void LocalStore::autoGC(bool sync) { -#ifdef HAVE_STATVFS +#if HAVE_STATVFS static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE"); auto getAvail = [this]() -> uint64_t { From 22ad0e653f9abff03de97fe928eaf13a99f7e8a1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 31 Jul 2024 22:14:27 +0200 Subject: [PATCH 106/142] Edit release notes --- doc/manual/src/release-notes/rl-2.24.md | 42 +++++++------------------ 1 file changed, 11 insertions(+), 31 deletions(-) diff --git a/doc/manual/src/release-notes/rl-2.24.md b/doc/manual/src/release-notes/rl-2.24.md index c68072573..cb82b1def 100644 --- a/doc/manual/src/release-notes/rl-2.24.md +++ b/doc/manual/src/release-notes/rl-2.24.md @@ -2,11 +2,11 @@ ### Significant changes -- Harden the user sandboxing +- Harden user sandboxing The build directory has been hardened against interference with the outside world by nesting it inside another directory owned by (and only readable by) the daemon user. - This is a low severity security fix, [CVE-2024-38531](https://www.cve.org/CVERecord?id=CVE-2024-38531), that was handled through the GitHub Security Advisories interface, and hence was merged directly in commit [2dd7f8f42](https://github.com/NixOS/nix/commit/2dd7f8f42da374d9fee4d424c1c6f82bcb36b393) instead of a PR. + This is a low severity security fix, [CVE-2024-38531](https://www.cve.org/CVERecord?id=CVE-2024-38531). Credit: [**@alois31**](https://github.com/alois31), [**Linus Heckemann (@lheckemann)**](https://github.com/lheckemann) Co-authors: [**@edolstra**](https://github.com/edolstra) @@ -99,7 +99,7 @@ Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) -- nix3-build: show all FOD errors with `--keep-going` [#10734](https://github.com/NixOS/nix/pull/10734) +- `nix build`: show all FOD errors with `--keep-going` [#10734](https://github.com/NixOS/nix/pull/10734) The [`nix build`](@docroot@/command-ref/new-cli/nix3-build.md) command has been updated to improve the behavior of the [`--keep-going`] flag. Now, when `--keep-going` is used, all hash-mismatch errors of failing fixed-output derivations (FODs) are displayed, similar to the behavior for other build failures. This enhancement ensures that all relevant build errors are shown, making it easier for users to update multiple derivations at once or to diagnose and fix issues. @@ -109,22 +109,20 @@ - Build with Meson [#2503](https://github.com/NixOS/nix/issues/2503) [#10378](https://github.com/NixOS/nix/pull/10378) [#10855](https://github.com/NixOS/nix/pull/10855) [#10904](https://github.com/NixOS/nix/pull/10904) [#10908](https://github.com/NixOS/nix/pull/10908) [#10914](https://github.com/NixOS/nix/pull/10914) [#10933](https://github.com/NixOS/nix/pull/10933) [#10936](https://github.com/NixOS/nix/pull/10936) [#10954](https://github.com/NixOS/nix/pull/10954) [#10955](https://github.com/NixOS/nix/pull/10955) [#10963](https://github.com/NixOS/nix/pull/10963) [#10967](https://github.com/NixOS/nix/pull/10967) [#10973](https://github.com/NixOS/nix/pull/10973) [#11034](https://github.com/NixOS/nix/pull/11034) [#11054](https://github.com/NixOS/nix/pull/11054) [#11055](https://github.com/NixOS/nix/pull/11055) [#11060](https://github.com/NixOS/nix/pull/11060) [#11064](https://github.com/NixOS/nix/pull/11064) [#11155](https://github.com/NixOS/nix/pull/11155) - These changes aim to replace the use of autotools and make with Meson for building various components of Nix. Additionally, each library is built in its own derivation, leveraging Meson's "subprojects" feature to allow a single development shell for building all libraries while also supporting separate builds. This approach aims to improve productivity and build modularity, compared to both make and a monolithic Meson-based derivation. + These changes aim to replace the use of autotools and `make` with Meson for building various components of Nix. Additionally, each library is built in its own derivation, leveraging Meson's "subprojects" feature to allow a single development shell for building all libraries while also supporting separate builds. This approach aims to improve productivity and build modularity, compared to both make and a monolithic Meson-based derivation. Special thanks to everyone who has contributed to the Meson port, particularly [**@p01arst0rm**](https://github.com/p01arst0rm) and [**@Qyriad**](https://github.com/Qyriad). Authors: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314), [**Tom Bereknyei**](https://github.com/tomberek), [**Théophane Hufschmitt (@thufschmitt)**](https://github.com/thufschmitt), [**Valentin Gagarin (@fricklerhandwerk)**](https://github.com/fricklerhandwerk), [**Robert Hensing (@roberth)**](https://github.com/roberth) Co-authors: [**@p01arst0rm**](https://github.com/p01arst0rm), [**@Qyriad**](https://github.com/Qyriad) -- Eval cache: fix cache regressions [#10570](https://github.com/NixOS/nix/issues/10570) [#11086](https://github.com/NixOS/nix/pull/11086) +- Evaluation cache: fix cache regressions [#10570](https://github.com/NixOS/nix/issues/10570) [#11086](https://github.com/NixOS/nix/pull/11086) This update addresses two bugs in the evaluation cache system: - 1. Regression in #10570: The evaluation cache was not being persisted in `nix develop` because `evalCaches` retained references to the caches and was never freed. + 1. Regression in #10570: The evaluation cache was not being persisted in `nix develop`. 2. Nix could sometimes try to commit the evaluation cache SQLite transaction without there being an active transaction, resulting in non-error errors being printed. - These bug fixes ensure that the evaluation cache is correctly managed and errors are appropriately handled. - Author: [**Lexi Mattick (@kognise)**](https://github.com/kognise) - Introduce `libnixflake` [#9063](https://github.com/NixOS/nix/pull/9063) @@ -137,9 +135,9 @@ Author: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314) -- CL options `--arg-from-file` and `--arg-from-stdin` [#9913](https://github.com/NixOS/nix/pull/9913) +- CLI options `--arg-from-file` and `--arg-from-stdin` [#9913](https://github.com/NixOS/nix/pull/9913) - The `--debugger` now prints source location information, instead of the +- The `--debugger` now prints source location information, instead of the pointers of source location information. Before: ``` @@ -160,11 +158,7 @@ 133| in ``` -- Make `nix store gc` use the auto-GC policy [#7851](https://github.com/NixOS/nix/pull/7851) - - - -- Stop vendoring toml11 +- Stop vendoring `toml11` We don't apply any patches to it, and vendoring it locks users into bugs (it hasn't been updated since its introduction in late 2021). @@ -176,7 +170,7 @@ Hash format `base32` was renamed to `nix32` since it used a special nix-specific character set for [Base32](https://en.wikipedia.org/wiki/Base32). - ## Deprecation: Use `nix32` instead of `base32` as `toHashFormat` + **Deprecation**: Use `nix32` instead of `base32` as `toHashFormat` For the builtin `convertHash`, the `toHashFormat` parameter now accepts the same hash formats as the `--to`/`--from` parameters of the `nix hash conert` command: `"base16"`, `"nix32"`, `"base64"`, and `"sri"`. The former `"base32"` value @@ -233,7 +227,7 @@ hello ``` - Older versions of `nix-shell` would resolve `shell.nix` relative to the current working directory; home in this example: + Older versions of `nix-shell` would resolve `shell.nix` relative to the current working directory, such as the user's home directory in this example: ```console [hostname:~]$ ./myproject/say-hi @@ -259,20 +253,6 @@ The old behavior can be opted into by setting the option [`nix-shell-shebang-arguments-relative-to-script`](@docroot@/command-ref/conf-file.md#conf-nix-shell-shebang-arguments-relative-to-script) to `false`. This option will be removed in a future release. - **`nix` command shebang** - - The experimental [`nix` command shebang](@docroot@/command-ref/new-cli/nix.md?highlight=shebang#shebang-interpreter) already behaves in this script-relative manner. - - Example: - - ```shell - #!/usr/bin/env nix - #!nix develop - #!nix --expr ``import ./shell.nix`` - #!nix -c bash - hello - ``` - Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) - Improve handling of tarballs that don't consist of a single top-level directory [#11195](https://github.com/NixOS/nix/pull/11195) From f136ec5290128470244006579d935653df355bdf Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 31 Jul 2024 22:16:44 +0200 Subject: [PATCH 107/142] Add contributors --- doc/manual/src/release-notes/rl-2.24.md | 46 ++++++++++++++++++- .../data/release-credits-email-to-handle.json | 3 +- .../data/release-credits-handle-to-name.json | 3 +- 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/release-notes/rl-2.24.md b/doc/manual/src/release-notes/rl-2.24.md index cb82b1def..5479cd3b9 100644 --- a/doc/manual/src/release-notes/rl-2.24.md +++ b/doc/manual/src/release-notes/rl-2.24.md @@ -277,4 +277,48 @@ # Contributors -Querying GitHub API for ee86e7f361c55c8c7dc2e45c3868802af249aeff, to get handle for MostAwesomeDude@gmail.com +This release was made possible by the following 43 contributors: + +- Andreas Rammhold [**(@andir)**](https://github.com/andir) +- Andrew Marshall [**(@amarshall)**](https://github.com/amarshall) +- Brian McKenna [**(@puffnfresh)**](https://github.com/puffnfresh) +- Cameron [**(@SkamDart)**](https://github.com/SkamDart) +- Cole Helbling [**(@cole-h)**](https://github.com/cole-h) +- Corbin Simpson [**(@MostAwesomeDude)**](https://github.com/MostAwesomeDude) +- Eelco Dolstra [**(@edolstra)**](https://github.com/edolstra) +- Emily [**(@emilazy)**](https://github.com/emilazy) +- Enno Richter [**(@elohmeier)**](https://github.com/elohmeier) +- Farid Zakaria [**(@fzakaria)**](https://github.com/fzakaria) +- HaeNoe [**(@haenoe)**](https://github.com/haenoe) +- Hamir Mahal [**(@hamirmahal)**](https://github.com/hamirmahal) +- Harmen [**(@alicebob)**](https://github.com/alicebob) +- Ivan Trubach [**(@tie)**](https://github.com/tie) +- Jared Baur [**(@jmbaur)**](https://github.com/jmbaur) +- John Ericson [**(@Ericson2314)**](https://github.com/Ericson2314) +- Jonathan De Troye [**(@detroyejr)**](https://github.com/detroyejr) +- Jörg Thalheim [**(@Mic92)**](https://github.com/Mic92) +- Klemens Nanni [**(@klemensn)**](https://github.com/klemensn) +- Las Safin [**(@L-as)**](https://github.com/L-as) +- Lexi Mattick [**(@kognise)**](https://github.com/kognise) +- Matthew Bauer [**(@matthewbauer)**](https://github.com/matthewbauer) +- Max “Goldstein” Siling [**(@GoldsteinE)**](https://github.com/GoldsteinE) +- Mingye Wang [**(@Artoria2e5)**](https://github.com/Artoria2e5) +- Philip Taron [**(@philiptaron)**](https://github.com/philiptaron) +- Pierre Bourdon [**(@delroth)**](https://github.com/delroth) +- Pino Toscano [**(@pinotree)**](https://github.com/pinotree) +- RTUnreal [**(@RTUnreal)**](https://github.com/RTUnreal) +- Robert Hensing [**(@roberth)**](https://github.com/roberth) +- Romain Neil [**(@romain-neil)**](https://github.com/romain-neil) +- Ryan Hendrickson [**(@rhendric)**](https://github.com/rhendric) +- Sergei Trofimovich [**(@trofi)**](https://github.com/trofi) +- Shogo Takata [**(@pineapplehunter)**](https://github.com/pineapplehunter) +- Siddhant Kumar [**(@siddhantk232)**](https://github.com/siddhantk232) +- Silvan Mosberger [**(@infinisil)**](https://github.com/infinisil) +- Théophane Hufschmitt [**(@thufschmitt)**](https://github.com/thufschmitt) +- Valentin Gagarin [**(@fricklerhandwerk)**](https://github.com/fricklerhandwerk) +- Winter [**(@winterqt)**](https://github.com/winterqt) +- jade [**(@lf-)**](https://github.com/lf-) +- kirillrdy [**(@kirillrdy)**](https://github.com/kirillrdy) +- pennae [**(@pennae)**](https://github.com/pennae) +- poweredbypie [**(@poweredbypie)**](https://github.com/poweredbypie) +- tomberek [**(@tomberek)**](https://github.com/tomberek) diff --git a/maintainers/data/release-credits-email-to-handle.json b/maintainers/data/release-credits-email-to-handle.json index 573ec2b31..cddc1a6e7 100644 --- a/maintainers/data/release-credits-email-to-handle.json +++ b/maintainers/data/release-credits-email-to-handle.json @@ -47,5 +47,6 @@ "pennae@lix.systems": "pennae", "delroth@gmail.com": "delroth", "enno@nerdworks.de": "elohmeier", - "mjbauer95@gmail.com": "matthewbauer" + "mjbauer95@gmail.com": "matthewbauer", + "MostAwesomeDude@gmail.com": "MostAwesomeDude" } \ No newline at end of file diff --git a/maintainers/data/release-credits-handle-to-name.json b/maintainers/data/release-credits-handle-to-name.json index d68311dde..abf9ed05b 100644 --- a/maintainers/data/release-credits-handle-to-name.json +++ b/maintainers/data/release-credits-handle-to-name.json @@ -40,5 +40,6 @@ "siddhantk232": "Siddhant Kumar", "winterqt": "Winter", "GoldsteinE": "Max \u201cGoldstein\u201d Siling", - "pennae": null + "pennae": null, + "MostAwesomeDude": "Corbin Simpson" } \ No newline at end of file From 8ff169715dde088a4c286b3379d3f548eafe3221 Mon Sep 17 00:00:00 2001 From: Qyriad Date: Mon, 29 Apr 2024 08:15:16 -0600 Subject: [PATCH 108/142] docs: clarify how the different kinds of installables are selected Change-Id: I146736bb97ebe035e04be69ce9fb60a557e38c6c --- src/nix/nix.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/nix/nix.md b/src/nix/nix.md index f958ce09a..34f73d032 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -59,9 +59,13 @@ These are command line arguments that represent something that can be realised i The following types of installable are supported by most commands: - [Flake output attribute](#flake-output-attribute) (experimental) + - This is the default - [Store path](#store-path) + - This is assumed if the argument is a Nix store path or a symlink to a Nix store path - [Nix file](#nix-file), optionally qualified by an attribute path + - Specified with `--file`/`-f` - [Nix expression](#nix-expression), optionally qualified by an attribute path + - Specified with `--expr`/`-E` For most commands, if no installable is specified, `.` is assumed. That is, Nix will operate on the default flake output attribute of the flake in the current directory. From cb5a5dd4f3064e3235fa908a44a5d074ef3c4205 Mon Sep 17 00:00:00 2001 From: Qyriad Date: Mon, 29 Apr 2024 08:09:50 -0600 Subject: [PATCH 109/142] docs: clarify how ^ works for -E/-f installables We didn't even realize you *could* use this syntax with -E and -f, much less that the attribute path could be *empty*. Change-Id: Id1a6715609f3a76a5ce477bd43a7832effbbe07b --- src/nix/nix.md | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/nix/nix.md b/src/nix/nix.md index 34f73d032..56587d0b2 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -182,9 +182,10 @@ that contains programs, and a `dev` output that provides development artifacts like C/C++ header files. The outputs on which `nix` commands operate are determined as follows: -* You can explicitly specify the desired outputs using the syntax - *installable*`^`*output1*`,`*...*`,`*outputN*. For example, you can - obtain the `dev` and `static` outputs of the `glibc` package: +* You can explicitly specify the desired outputs using the syntax *installable*`^`*output1*`,`*...*`,`*outputN* — that is, a caret followed immediately by a comma-separated list of derivation outputs to select. + For installables specified as [Flake output attributes](#flake-output-attribute) or [Store paths](#store-path), the output is specified in the same argument: + + For example, you can obtain the `dev` and `static` outputs of the `glibc` package: ```console # nix build 'nixpkgs#glibc^dev,static' @@ -199,6 +200,19 @@ operate are determined as follows: … ``` + For `-e`/`--expr` and `-f`/`--file`, the derivation output is specified as part of the attribute path: + + ```console + $ nix build -f '' 'glibc^dev,static' + $ nix build --impure -E 'import { }' 'glibc^dev,static' + ``` + + This syntax is the same even if the actual attribute path is empty: + + ```console + $ nix build -E 'let pkgs = import { }; in pkgs.glibc' '^dev,static' + ``` + * You can also specify that *all* outputs should be used using the syntax *installable*`^*`. For example, the following shows the size of all outputs of the `glibc` package in the binary cache: From 794a50065b33cbaaaf1f6ff9dbf954bb5eedfc14 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 31 Jul 2024 22:33:41 +0200 Subject: [PATCH 110/142] base32 -> nix32 --- doc/manual/src/release-notes/rl-2.24.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/release-notes/rl-2.24.md b/doc/manual/src/release-notes/rl-2.24.md index 5479cd3b9..5bcc1d79c 100644 --- a/doc/manual/src/release-notes/rl-2.24.md +++ b/doc/manual/src/release-notes/rl-2.24.md @@ -174,7 +174,7 @@ For the builtin `convertHash`, the `toHashFormat` parameter now accepts the same hash formats as the `--to`/`--from` parameters of the `nix hash conert` command: `"base16"`, `"nix32"`, `"base64"`, and `"sri"`. The former `"base32"` value - remains as a deprecated alias for `"base32"`. Please convert your code from: + remains as a deprecated alias for `"nix32"`. Please convert your code from: ```nix builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base32";} From 6ed67d35ed51905434eba7d4a167b43aa581478f Mon Sep 17 00:00:00 2001 From: Ryan Hendrickson Date: Wed, 31 Jul 2024 17:39:43 -0400 Subject: [PATCH 111/142] docs: add variables; rework scope (#11062) Co-authored-by: Valentin Gagarin --- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/language/identifiers.md | 2 +- doc/manual/src/language/scope.md | 32 ++++++++++++++++++-------- doc/manual/src/language/variables.md | 10 ++++++++ 4 files changed, 35 insertions(+), 10 deletions(-) create mode 100644 doc/manual/src/language/variables.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 8739599a0..7661f5f62 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -28,6 +28,7 @@ - [Data Types](language/types.md) - [String context](language/string-context.md) - [Syntax and semantics](language/syntax.md) + - [Variables](language/variables.md) - [Identifiers](language/identifiers.md) - [Scoping rules](language/scope.md) - [String interpolation](language/string-interpolation.md) diff --git a/doc/manual/src/language/identifiers.md b/doc/manual/src/language/identifiers.md index c9e981da6..bd58a9b36 100644 --- a/doc/manual/src/language/identifiers.md +++ b/doc/manual/src/language/identifiers.md @@ -22,7 +22,7 @@ A name can be an [identifier](#identifier) or a [string literal](./syntax.md#str > > *name* → *identifier* | *string* -Names are used in [attribute sets](./syntax.md#attrs-literal), [`let` bindings](./syntax.md#let-expressions), and [`inherit`](./syntax.md#inheriting attributes). +Names are used in [attribute sets](./syntax.md#attrs-literal), [`let` bindings](./syntax.md#let-expressions), and [`inherit`](./syntax.md#inheriting-attributes). # Keywords diff --git a/doc/manual/src/language/scope.md b/doc/manual/src/language/scope.md index 5c6aed38d..96c7468eb 100644 --- a/doc/manual/src/language/scope.md +++ b/doc/manual/src/language/scope.md @@ -1,14 +1,28 @@ # Scoping rules -Nix is [statically scoped](https://en.wikipedia.org/wiki/Scope_(computer_science)#Lexical_scope), but with multiple scopes and shadowing rules. +A *scope* in the Nix language is a dictionary keyed by [name](./identifiers.md#names), mapping each name to an expression and a *definition type*. +The definition type is either *explicit* or *implicit*. +Each entry in this dictionary is a *definition*. -* primary scope: explicitly-bound variables - * [`let`](./syntax.md#let-expressions) - * [`inherit`](./syntax.md#inheriting-attributes) - * [function](./syntax.md#functions) arguments +Explicit definitions are created by the following expressions: +- [let-expressions](syntax.md#let-expressions) +- [recursive attribute set literals](syntax.md#recursive-sets) (`rec`) +- [function literals](syntax.md#functions) -* secondary scope: implicitly-bound variables - * [`with`](./syntax.md#with-expressions) +Implicit definitions are only created by [with-expressions](./syntax.md#with-expressions). -Primary scope takes precedence over secondary scope. -See [`with`](./syntax.md#with-expressions) for a detailed example. +Every expression is *enclosed* by a scope. +The outermost expression is enclosed by the [built-in, global scope](./builtins.md), which contains only explicit definitions. +The respective definition types *extend* their enclosing scope by adding new definitions, or replacing existing ones with the same name. +An explicit definition can replace a definition of any type; an implicit definition can only replace another implicit definition. + +Each of the above expressions defines which of its subexpressions are enclosed by the extended scope. +In all other cases, the same scope that encloses an expression is the enclosing scope for its subexpressions. + +The Nix language is [statically scoped](https://en.wikipedia.org/wiki/Scope_(computer_science)#Lexical_scope); +the value of a variable is determined only by the variable's enclosing scope, and not by the dynamic context in which the variable is evaluated. + +> **Note** +> +> Expressions entered into the [Nix REPL](@docroot@/command-ref/new-cli/nix3-repl.md) are enclosed by a scope that can be extended by command line arguments or previous REPL commands. +> These ways of extending scope are not, strictly speaking, part of the Nix language. diff --git a/doc/manual/src/language/variables.md b/doc/manual/src/language/variables.md new file mode 100644 index 000000000..af6aff8a2 --- /dev/null +++ b/doc/manual/src/language/variables.md @@ -0,0 +1,10 @@ +# Variables + +A *variable* is an [identifier](identifiers.md) used as an expression. + +> **Syntax** +> +> *expression* → *identifier* + +A variable must have the same name as a definition in the [scope](./scope.md) that encloses it. +The value of a variable is the value of the corresponding expression in the enclosing scope. From 9e8afc68e554c5c45b8e9f387d4fcf4ac0174a6a Mon Sep 17 00:00:00 2001 From: Ryan Hendrickson Date: Wed, 31 Jul 2024 19:07:57 -0400 Subject: [PATCH 112/142] docs: add language/string-literals.md --- doc/manual/redirects.js | 1 + doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/language/identifiers.md | 2 +- .../src/language/string-interpolation.md | 4 + doc/manual/src/language/string-literals.md | 189 ++++++++++++++++++ doc/manual/src/language/syntax.md | 172 +--------------- doc/manual/src/language/types.md | 2 +- 7 files changed, 199 insertions(+), 172 deletions(-) create mode 100644 doc/manual/src/language/string-literals.md diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index beef6ef4a..cb8cd18fa 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -344,6 +344,7 @@ const redirects = { }, "language/syntax.html": { "scoping-rules": "scoping.html", + "string-literal": "string-literals.html", }, "installation/installing-binary.html": { "linux": "uninstall.html#linux", diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 7661f5f62..eef7d189c 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -29,6 +29,7 @@ - [String context](language/string-context.md) - [Syntax and semantics](language/syntax.md) - [Variables](language/variables.md) + - [String literals](language/string-literals.md) - [Identifiers](language/identifiers.md) - [Scoping rules](language/scope.md) - [String interpolation](language/string-interpolation.md) diff --git a/doc/manual/src/language/identifiers.md b/doc/manual/src/language/identifiers.md index bd58a9b36..a4153b588 100644 --- a/doc/manual/src/language/identifiers.md +++ b/doc/manual/src/language/identifiers.md @@ -16,7 +16,7 @@ An *identifier* is an [ASCII](https://en.wikipedia.org/wiki/ASCII) character seq # Names -A name can be an [identifier](#identifier) or a [string literal](./syntax.md#string-literal). +A name can be an [identifier](#identifier) or a [string literal](string-literals.md). > **Syntax** > diff --git a/doc/manual/src/language/string-interpolation.md b/doc/manual/src/language/string-interpolation.md index 1778bdfa0..27780dcbb 100644 --- a/doc/manual/src/language/string-interpolation.md +++ b/doc/manual/src/language/string-interpolation.md @@ -8,6 +8,10 @@ Such a construct is called *interpolated string*, and the expression inside is a [path]: ./types.md#type-path [attribute set]: ./types.md#attribute-set +> **Syntax** +> +> *interpolation_element* → `${` *expression* `}` + ## Examples ### String diff --git a/doc/manual/src/language/string-literals.md b/doc/manual/src/language/string-literals.md new file mode 100644 index 000000000..7605e8c61 --- /dev/null +++ b/doc/manual/src/language/string-literals.md @@ -0,0 +1,189 @@ +# String literals + +A *string literal* represents a [string](types.md#type-string) value. + +> **Syntax** +> +> *expression* → *string* +> +> *string* → `"` ( *string_char*\* [*interpolation_element*][string interpolation] )* *string_char*\* `"` +> +> *string* → `''` ( *indented_string_char*\* [*interpolation_element*][string interpolation] )* *indented_string_char*\* `''` +> +> *string* → *uri* +> +> *string_char* ~ `[^"$\\]|\$(?!\{)|\\.` +> +> *indented_string_char* ~ `[^$']|\$\$|\$(?!\{)|''[$']|''\\.|'(?!')` +> +> *uri* ~ `[A-Za-z][+\-.0-9A-Za-z]*:[!$%&'*+,\-./0-9:=?@A-Z_a-z~]+` + +Strings can be written in three ways. + +The most common way is to enclose the string between double quotes, e.g., `"foo bar"`. +Strings can span multiple lines. +The results of other expressions can be included into a string by enclosing them in `${ }`, a feature known as [string interpolation]. + +[string interpolation]: ./string-interpolation.md + +The following must be escaped to represent them within a string, by prefixing with a backslash (`\`): + +- Double quote (`"`) + +> **Example** +> +> ```nix +> "\"" +> ``` +> +> "\"" + +- Backslash (`\`) + +> **Example** +> +> ```nix +> "\\" +> ``` +> +> "\\" + +- Dollar sign followed by an opening curly bracket (`${`) – "dollar-curly" + +> **Example** +> +> ```nix +> "\${" +> ``` +> +> "\${" + +The newline, carriage return, and tab characters can be written as `\n`, `\r` and `\t`, respectively. + +A "double-dollar-curly" (`$${`) can be written literally. + +> **Example** +> +> ```nix +> "$${" +> ``` +> +> "$\${" + +String values are output on the terminal with Nix-specific escaping. +Strings written to files will contain the characters encoded by the escaping. + +The second way to write string literals is as an *indented string*, which is enclosed between pairs of *double single-quotes* (`''`), like so: + +```nix +'' +This is the first line. +This is the second line. + This is the third line. +'' +``` + +This kind of string literal intelligently strips indentation from +the start of each line. To be precise, it strips from each line a +number of spaces equal to the minimal indentation of the string as a +whole (disregarding the indentation of empty lines). For instance, +the first and second line are indented two spaces, while the third +line is indented four spaces. Thus, two spaces are stripped from +each line, so the resulting string is + +```nix +"This is the first line.\nThis is the second line.\n This is the third line.\n" +``` + +> **Note** +> +> Whitespace and newline following the opening `''` is ignored if there is no non-whitespace text on the initial line. + +> **Warning** +> +> Prefixed tab characters are not stripped. +> +> > **Example** +> > +> > The following indented string is prefixed with tabs: +> > +> > '' +> > all: +> > @echo hello +> > '' +> > +> > "\tall:\n\t\t@echo hello\n" + +Indented strings support [string interpolation]. + +The following must be escaped to represent them in an indented string: + +- `$` is escaped by prefixing it with two single quotes (`''`) + +> **Example** +> +> ```nix +> '' +> ''$ +> '' +> ``` +> +> "$\n" + +- `''` is escaped by prefixing it with one single quote (`'`) + +> **Example** +> +> ```nix +> '' +> ''' +> '' +> ``` +> +> "''\n" + +These special characters are escaped as follows: +- Linefeed (`\n`): `''\n` +- Carriage return (`\r`): `''\r` +- Tab (`\t`): `''\t` + +`''\` escapes any other character. + +A "double-dollar-curly" (`$${`) can be written literally. + +> **Example** +> +> ```nix +> '' +> $${ +> '' +> ``` +> +> "$\${\n" + +Indented strings are primarily useful in that they allow multi-line +string literals to follow the indentation of the enclosing Nix +expression, and that less escaping is typically necessary for +strings representing languages such as shell scripts and +configuration files because `''` is much less common than `"`. +Example: + +```nix +stdenv.mkDerivation { +... +postInstall = + '' + mkdir $out/bin $out/etc + cp foo $out/bin + echo "Hello World" > $out/etc/foo.conf + ${if enableBar then "cp bar $out/bin" else ""} + ''; +... +} +``` + +Finally, as a convenience, *URIs* as defined in appendix B of +[RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as +is*, without quotes. For instance, the string +`"http://example.org/foo.tar.bz2"` can also be written as +`http://example.org/foo.tar.bz2`. diff --git a/doc/manual/src/language/syntax.md b/doc/manual/src/language/syntax.md index 6108bacd6..daf073aef 100644 --- a/doc/manual/src/language/syntax.md +++ b/doc/manual/src/language/syntax.md @@ -6,175 +6,7 @@ This section covers syntax and semantics of the Nix language. ### String {#string-literal} - *Strings* can be written in three ways. - - The most common way is to enclose the string between double quotes, e.g., `"foo bar"`. - Strings can span multiple lines. - The results of other expressions can be included into a string by enclosing them in `${ }`, a feature known as [string interpolation]. - - [string interpolation]: ./string-interpolation.md - - The following must be escaped to represent them within a string, by prefixing with a backslash (`\`): - - - Double quote (`"`) - - > **Example** - > - > ```nix - > "\"" - > ``` - > - > "\"" - - - Backslash (`\`) - - > **Example** - > - > ```nix - > "\\" - > ``` - > - > "\\" - - - Dollar sign followed by an opening curly bracket (`${`) – "dollar-curly" - - > **Example** - > - > ```nix - > "\${" - > ``` - > - > "\${" - - The newline, carriage return, and tab characters can be written as `\n`, `\r` and `\t`, respectively. - - A "double-dollar-curly" (`$${`) can be written literally. - - > **Example** - > - > ```nix - > "$${" - > ``` - > - > "$\${" - - String values are output on the terminal with Nix-specific escaping. - Strings written to files will contain the characters encoded by the escaping. - - The second way to write string literals is as an *indented string*, which is enclosed between pairs of *double single-quotes* (`''`), like so: - - ```nix - '' - This is the first line. - This is the second line. - This is the third line. - '' - ``` - - This kind of string literal intelligently strips indentation from - the start of each line. To be precise, it strips from each line a - number of spaces equal to the minimal indentation of the string as a - whole (disregarding the indentation of empty lines). For instance, - the first and second line are indented two spaces, while the third - line is indented four spaces. Thus, two spaces are stripped from - each line, so the resulting string is - - ```nix - "This is the first line.\nThis is the second line.\n This is the third line.\n" - ``` - - > **Note** - > - > Whitespace and newline following the opening `''` is ignored if there is no non-whitespace text on the initial line. - - > **Warning** - > - > Prefixed tab characters are not stripped. - > - > > **Example** - > > - > > The following indented string is prefixed with tabs: - > > - > > '' - > > all: - > > @echo hello - > > '' - > > - > > "\tall:\n\t\t@echo hello\n" - - Indented strings support [string interpolation]. - - The following must be escaped to represent them in an indented string: - - - `$` is escaped by prefixing it with two single quotes (`''`) - - > **Example** - > - > ```nix - > '' - > ''$ - > '' - > ``` - > - > "$\n" - - - `''` is escaped by prefixing it with one single quote (`'`) - - > **Example** - > - > ```nix - > '' - > ''' - > '' - > ``` - > - > "''\n" - - These special characters are escaped as follows: - - Linefeed (`\n`): `''\n` - - Carriage return (`\r`): `''\r` - - Tab (`\t`): `''\t` - - `''\` escapes any other character. - - A "double-dollar-curly" (`$${`) can be written literally. - - > **Example** - > - > ```nix - > '' - > $${ - > '' - > ``` - > - > "$\${\n" - - Indented strings are primarily useful in that they allow multi-line - string literals to follow the indentation of the enclosing Nix - expression, and that less escaping is typically necessary for - strings representing languages such as shell scripts and - configuration files because `''` is much less common than `"`. - Example: - - ```nix - stdenv.mkDerivation { - ... - postInstall = - '' - mkdir $out/bin $out/etc - cp foo $out/bin - echo "Hello World" > $out/etc/foo.conf - ${if enableBar then "cp bar $out/bin" else ""} - ''; - ... - } - ``` - - Finally, as a convenience, *URIs* as defined in appendix B of - [RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as - is*, without quotes. For instance, the string - `"http://example.org/foo.tar.bz2"` can also be written as - `http://example.org/foo.tar.bz2`. +See [String literals](string-literals.md). ### Number {#number-literal} @@ -253,7 +85,7 @@ Attribute sets are written enclosed in curly brackets (`{ }`). Attribute names and attribute values are separated by an equal sign (`=`). Each value can be an arbitrary expression, terminated by a semicolon (`;`) -An attribute name is a string without context, and is denoted by a [name] (an [identifier](./identifiers.md#identifiers) or [string literal](#string-literal)). +An attribute name is a string without context, and is denoted by a [name] (an [identifier](./identifiers.md#identifiers) or [string literal](string-literals.md)). [name]: ./identifiers.md#names diff --git a/doc/manual/src/language/types.md b/doc/manual/src/language/types.md index 229756e6b..82184a8b0 100644 --- a/doc/manual/src/language/types.md +++ b/doc/manual/src/language/types.md @@ -45,7 +45,7 @@ The function [`builtins.isBool`](builtins.md#builtins-isBool) can be used to det A _string_ in the Nix language is an immutable, finite-length sequence of bytes, along with a [string context](string-context.md). Nix does not assume or support working natively with character encodings. -String values without string context can be expressed as [string literals](syntax.md#string-literal). +String values without string context can be expressed as [string literals](string-literals.md). The function [`builtins.isString`](builtins.md#builtins-isString) can be used to determine if a value is a string. ### Path {#type-path} From 17318bc70dd0a3712d0a2c94a180dd9f614c97b0 Mon Sep 17 00:00:00 2001 From: Ryan Hendrickson Date: Wed, 31 Jul 2024 19:22:17 -0400 Subject: [PATCH 113/142] docs: fix string literal example formatting --- doc/manual/src/language/string-literals.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/doc/manual/src/language/string-literals.md b/doc/manual/src/language/string-literals.md index 7605e8c61..8f4b75f3e 100644 --- a/doc/manual/src/language/string-literals.md +++ b/doc/manual/src/language/string-literals.md @@ -36,7 +36,7 @@ The following must be escaped to represent them within a string, by prefixing wi > "\"" > ``` > -> "\"" +> "\"" - Backslash (`\`) @@ -46,7 +46,7 @@ The following must be escaped to represent them within a string, by prefixing wi > "\\" > ``` > -> "\\" +> "\\" - Dollar sign followed by an opening curly bracket (`${`) – "dollar-curly" @@ -56,7 +56,7 @@ The following must be escaped to represent them within a string, by prefixing wi > "\${" > ``` > -> "\${" +> "\${" The newline, carriage return, and tab characters can be written as `\n`, `\r` and `\t`, respectively. @@ -68,7 +68,7 @@ A "double-dollar-curly" (`$${`) can be written literally. > "$${" > ``` > -> "$\${" +> "$\${" String values are output on the terminal with Nix-specific escaping. Strings written to files will contain the characters encoded by the escaping. @@ -107,10 +107,11 @@ each line, so the resulting string is > > > > The following indented string is prefixed with tabs: > > -> > '' +> > 
''
 > > 	all:
 > > 		@echo hello
 > > ''
+> >
> > > > "\tall:\n\t\t@echo hello\n" From 617e711820762d27285eb8710ce42d52f6448acd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 1 Aug 2024 10:41:42 +0200 Subject: [PATCH 114/142] 'build' is now 'build.nix' --- maintainers/upload-release.pl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 4c4e2bd6f..731988568 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -42,7 +42,7 @@ my $flakeUrl = $evalInfo->{flake}; my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die) if $flakeUrl; my $nixRev = ($flakeInfo ? $flakeInfo->{revision} : $evalInfo->{jobsetevalinputs}->{nix}->{revision}) or die; -my $buildInfo = decode_json(fetch("$evalUrl/job/build.x86_64-linux", 'application/json')); +my $buildInfo = decode_json(fetch("$evalUrl/job/build.nix.x86_64-linux", 'application/json')); #print Dumper($buildInfo); my $releaseName = $buildInfo->{nixname}; @@ -91,7 +91,7 @@ sub getStorePath { sub copyManual { my $manual; eval { - $manual = getStorePath("build.x86_64-linux", "doc"); + $manual = getStorePath("build.nix.x86_64-linux", "doc"); }; if ($@) { warn "$@"; @@ -240,12 +240,12 @@ if ($haveDocker) { # Upload nix-fallback-paths.nix. write_file("$tmpDir/fallback-paths.nix", "{\n" . - " x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" . - " i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" . - " aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" . - " riscv64-linux = \"" . getStorePath("buildCross.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" . - " x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" . - " aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" . + " x86_64-linux = \"" . getStorePath("build.nix.x86_64-linux") . "\";\n" . + " i686-linux = \"" . getStorePath("build.nix.i686-linux") . "\";\n" . + " aarch64-linux = \"" . getStorePath("build.nix.aarch64-linux") . "\";\n" . + " riscv64-linux = \"" . getStorePath("buildCross.nix.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" . + " x86_64-darwin = \"" . getStorePath("build.nix.x86_64-darwin") . "\";\n" . + " aarch64-darwin = \"" . getStorePath("build.nix.aarch64-darwin") . "\";\n" . "}\n"); # Upload release files to S3. From 30aca6f243d8d5dcdbbfcf2a36bcdefa9d2aeeee Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 1 Aug 2024 10:43:00 +0200 Subject: [PATCH 115/142] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index ad2261920..5c18f9195 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.24.0 +2.25.0 From b291b61089da1121145d92d6747ad2b18737ad9f Mon Sep 17 00:00:00 2001 From: Ryan Hendrickson Date: Thu, 1 Aug 2024 05:14:49 -0400 Subject: [PATCH 116/142] docs: editorial quibbles (#11232) --- doc/manual/src/language/identifiers.md | 3 ++- doc/manual/src/language/scope.md | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/language/identifiers.md b/doc/manual/src/language/identifiers.md index bd58a9b36..861ee3e20 100644 --- a/doc/manual/src/language/identifiers.md +++ b/doc/manual/src/language/identifiers.md @@ -16,13 +16,14 @@ An *identifier* is an [ASCII](https://en.wikipedia.org/wiki/ASCII) character seq # Names -A name can be an [identifier](#identifier) or a [string literal](./syntax.md#string-literal). +A *name* can be written as an [identifier](#identifier) or a [string literal](./syntax.md#string-literal). > **Syntax** > > *name* → *identifier* | *string* Names are used in [attribute sets](./syntax.md#attrs-literal), [`let` bindings](./syntax.md#let-expressions), and [`inherit`](./syntax.md#inheriting-attributes). +Two names are the same if they represent the same sequence of characters, regardless of whether they are written as identifiers or strings. # Keywords diff --git a/doc/manual/src/language/scope.md b/doc/manual/src/language/scope.md index 96c7468eb..9373324e2 100644 --- a/doc/manual/src/language/scope.md +++ b/doc/manual/src/language/scope.md @@ -13,7 +13,7 @@ Implicit definitions are only created by [with-expressions](./syntax.md#with-exp Every expression is *enclosed* by a scope. The outermost expression is enclosed by the [built-in, global scope](./builtins.md), which contains only explicit definitions. -The respective definition types *extend* their enclosing scope by adding new definitions, or replacing existing ones with the same name. +The expressions listed above *extend* their enclosing scope by adding new definitions, or replacing existing ones with the same name. An explicit definition can replace a definition of any type; an implicit definition can only replace another implicit definition. Each of the above expressions defines which of its subexpressions are enclosed by the extended scope. From 9b5b7b796341eca437fe08bb278c49dfbae2deaa Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 1 Aug 2024 16:51:57 +0200 Subject: [PATCH 117/142] Fix the S3 store It was failing with: error: AWS error fetching 'nix-cache-info': The specified bucket does not exist because `S3BinaryCacheStoreImpl` had a `bucketName` field that shadowed the inherited `bucketName from `S3BinaryCacheStoreConfig`. --- src/libstore/s3-binary-cache-store.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 92ab47cd6..21175b1eb 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -220,8 +220,6 @@ std::string S3BinaryCacheStoreConfig::doc() struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStore { - std::string bucketName; - Stats stats; S3Helper s3Helper; From 739418504c4d2f28fb5f45151b1c83707c3571e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 2 Aug 2024 11:12:06 +0200 Subject: [PATCH 118/142] allow to c api with older c versions In the FFI world we have many tools that are not gcc/clang and therefore not always support the latest C standard. This fixes support with cffi i.e. used in https://github.com/tweag/python-nix --- src/libexpr-c/nix_api_expr.h | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/libexpr-c/nix_api_expr.h b/src/libexpr-c/nix_api_expr.h index adf8b65b1..1764b49f3 100644 --- a/src/libexpr-c/nix_api_expr.h +++ b/src/libexpr-c/nix_api_expr.h @@ -14,6 +14,16 @@ #include "nix_api_util.h" #include +#ifndef __has_c_attribute +# define __has_c_attribute(x) 0 +#endif + +#if __has_c_attribute(deprecated) +# define NIX_DEPRECATED(msg) [[deprecated(msg)]] +#else +# define NIX_DEPRECATED(msg) +#endif + #ifdef __cplusplus extern "C" { #endif @@ -45,7 +55,7 @@ typedef struct EvalState EvalState; // nix::EvalState * @see nix_value_incref, nix_value_decref */ typedef struct nix_value nix_value; -[[deprecated("use nix_value instead")]] typedef nix_value Value; +NIX_DEPRECATED("use nix_value instead") typedef nix_value Value; // Function prototypes /** From 5a6e28e1667386b6ce4310bb07ed196432d7a9ec Mon Sep 17 00:00:00 2001 From: a-kenji Date: Mon, 5 Aug 2024 01:03:21 +0200 Subject: [PATCH 119/142] docs: installable remove alternate expression flag (#11254) --- src/nix/nix.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/nix/nix.md b/src/nix/nix.md index 56587d0b2..443757a05 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -65,7 +65,7 @@ The following types of installable are supported by most commands: - [Nix file](#nix-file), optionally qualified by an attribute path - Specified with `--file`/`-f` - [Nix expression](#nix-expression), optionally qualified by an attribute path - - Specified with `--expr`/`-E` + - Specified with `--expr` For most commands, if no installable is specified, `.` is assumed. That is, Nix will operate on the default flake output attribute of the flake in the current directory. @@ -200,17 +200,17 @@ operate are determined as follows: … ``` - For `-e`/`--expr` and `-f`/`--file`, the derivation output is specified as part of the attribute path: + For `--expr` and `-f`/`--file`, the derivation output is specified as part of the attribute path: ```console $ nix build -f '' 'glibc^dev,static' - $ nix build --impure -E 'import { }' 'glibc^dev,static' + $ nix build --impure --expr 'import { }' 'glibc^dev,static' ``` This syntax is the same even if the actual attribute path is empty: ```console - $ nix build -E 'let pkgs = import { }; in pkgs.glibc' '^dev,static' + $ nix build --impure --expr 'let pkgs = import { }; in pkgs.glibc' '^dev,static' ``` * You can also specify that *all* outputs should be used using the From 2950f9e18af1bd57b566b8c0b4df71022edb3b80 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 5 Aug 2024 11:38:38 +0200 Subject: [PATCH 120/142] Add a VM test for S3BinaryCacheStore Fixes #11238. --- tests/nixos/default.nix | 2 + tests/nixos/nix-copy-closure.nix | 2 +- tests/nixos/s3-binary-cache-store.nix | 63 +++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 tests/nixos/s3-binary-cache-store.nix diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index c0c7b42fd..66174c928 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -146,4 +146,6 @@ in functional_root = runNixOSTestFor "x86_64-linux" ./functional/as-root.nix; user-sandboxing = runNixOSTestFor "x86_64-linux" ./user-sandboxing; + + s3-binary-cache-store = runNixOSTestFor "x86_64-linux" ./s3-binary-cache-store.nix; } diff --git a/tests/nixos/nix-copy-closure.nix b/tests/nixos/nix-copy-closure.nix index 66cbfb033..b9daa0a1f 100644 --- a/tests/nixos/nix-copy-closure.nix +++ b/tests/nixos/nix-copy-closure.nix @@ -1,6 +1,6 @@ # Test ‘nix-copy-closure’. -{ lib, config, nixpkgs, hostPkgs, ... }: +{ lib, config, nixpkgs, ... }: let pkgs = config.nodes.client.nixpkgs.pkgs; diff --git a/tests/nixos/s3-binary-cache-store.nix b/tests/nixos/s3-binary-cache-store.nix new file mode 100644 index 000000000..015457968 --- /dev/null +++ b/tests/nixos/s3-binary-cache-store.nix @@ -0,0 +1,63 @@ +{ lib, config, nixpkgs, ... }: + +let + pkgs = config.nodes.client.nixpkgs.pkgs; + + pkgA = pkgs.cowsay; + + accessKey = "BKIKJAA5BMMU2RHO6IBB"; + secretKey = "V7f1CwQqAcwo80UEIJEjc5gVQUSSx5ohQ9GSrr12"; + env = "AWS_ACCESS_KEY_ID=${accessKey} AWS_SECRET_ACCESS_KEY=${secretKey}"; + + storeUrl = "s3://my-cache?endpoint=http://server:9000®ion=eu-west-1"; + +in { + name = "nix-copy-closure"; + + nodes = + { server = + { config, lib, pkgs, ... }: + { virtualisation.writableStore = true; + virtualisation.additionalPaths = [ pkgA ]; + environment.systemPackages = [ pkgs.minio-client ]; + nix.extraOptions = "experimental-features = nix-command"; + services.minio = { + enable = true; + region = "eu-west-1"; + rootCredentialsFile = pkgs.writeText "minio-credentials-full" '' + MINIO_ROOT_USER=${accessKey} + MINIO_ROOT_PASSWORD=${secretKey} + ''; + }; + networking.firewall.allowedTCPPorts = [ 9000 ]; + }; + + client = + { config, pkgs, ... }: + { virtualisation.writableStore = true; + nix.extraOptions = "experimental-features = nix-command"; + }; + }; + + testScript = { nodes }: '' + # fmt: off + start_all() + + # Create a binary cache. + server.wait_for_unit("minio") + + server.succeed("mc config host add minio http://localhost:9000 ${accessKey} ${secretKey} --api s3v4") + server.succeed("mc mb minio/my-cache") + + server.succeed("${env} nix copy --to '${storeUrl}' ${pkgA}") + + # Copy a package from the binary cache. + client.fail("nix path-info ${pkgA}") + + client.succeed("${env} nix store info --store '${storeUrl}' >&2") + + client.succeed("${env} nix copy --no-check-sigs --from '${storeUrl}' ${pkgA}") + + client.succeed("nix path-info ${pkgA}") + ''; +} From 9d2d4d11e6fad32d97cb3605614835a16c5b5cdf Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 5 Aug 2024 12:34:05 -0400 Subject: [PATCH 121/142] Split tests, organize more string functions The test split matches PR #8920, so the utility files and tests files are once again to 1-1. The string changes continues what was started in PR #11093. --- src/libexpr/eval-cache.cc | 2 + src/libutil/strings-inline.hh | 41 ++ src/libutil/strings.cc | 13 +- src/libutil/strings.hh | 28 ++ src/libutil/util.cc | 18 - src/libutil/util.hh | 44 +- tests/unit/libutil/file-system.cc | 258 +++++++++++ tests/unit/libutil/meson.build | 5 +- tests/unit/libutil/processes.cc | 17 + tests/unit/libutil/strings.cc | 151 +++++++ tests/unit/libutil/terminal.cc | 60 +++ tests/unit/libutil/tests.cc | 703 ------------------------------ tests/unit/libutil/util.cc | 385 ++++++++++++++++ 13 files changed, 959 insertions(+), 766 deletions(-) create mode 100644 tests/unit/libutil/file-system.cc create mode 100644 tests/unit/libutil/processes.cc create mode 100644 tests/unit/libutil/terminal.cc delete mode 100644 tests/unit/libutil/tests.cc create mode 100644 tests/unit/libutil/util.cc diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 26352187e..3d77c029e 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -4,6 +4,8 @@ #include "eval.hh" #include "eval-inline.hh" #include "store-api.hh" +// Need specialization involving `SymbolStr` just in this one module. +#include "strings-inline.hh" namespace nix::eval_cache { diff --git a/src/libutil/strings-inline.hh b/src/libutil/strings-inline.hh index 10c1b19e6..d254d486d 100644 --- a/src/libutil/strings-inline.hh +++ b/src/libutil/strings-inline.hh @@ -4,6 +4,21 @@ namespace nix { +template +C tokenizeString(std::string_view s, std::string_view separators) +{ + C result; + auto pos = s.find_first_not_of(separators, 0); + while (pos != s.npos) { + auto end = s.find_first_of(separators, pos + 1); + if (end == s.npos) + end = s.size(); + result.insert(result.end(), std::string(s, pos, end - pos)); + pos = s.find_first_not_of(separators, end); + } + return result; +} + template std::string concatStringsSep(const std::string_view sep, const C & ss) { @@ -28,4 +43,30 @@ std::string concatStringsSep(const std::string_view sep, const C & ss) return s; } +template +std::string dropEmptyInitThenConcatStringsSep(const std::string_view sep, const C & ss) +{ + size_t size = 0; + + // TODO? remove to make sure we don't rely on the empty item ignoring behavior, + // or just get rid of this function by understanding the remaining calls. + // for (auto & i : ss) { + // // Make sure we don't rely on the empty item ignoring behavior + // assert(!i.empty()); + // break; + // } + + // need a cast to string_view since this is also called with Symbols + for (const auto & s : ss) + size += sep.size() + std::string_view(s).size(); + std::string s; + s.reserve(size); + for (auto & i : ss) { + if (s.size() != 0) + s += sep; + s += i; + } + return s; +} + } // namespace nix diff --git a/src/libutil/strings.cc b/src/libutil/strings.cc index 7ec618bf4..2bb7f8c0a 100644 --- a/src/libutil/strings.cc +++ b/src/libutil/strings.cc @@ -1,12 +1,15 @@ #include #include "strings-inline.hh" -#include "util.hh" namespace nix { -template std::string concatStringsSep(std::string_view, const Strings &); -template std::string concatStringsSep(std::string_view, const StringSet &); +template std::list tokenizeString(std::string_view s, std::string_view separators); +template std::set tokenizeString(std::string_view s, std::string_view separators); +template std::vector tokenizeString(std::string_view s, std::string_view separators); + +template std::string concatStringsSep(std::string_view, const std::list &); +template std::string concatStringsSep(std::string_view, const std::set &); template std::string concatStringsSep(std::string_view, const std::vector &); typedef std::string_view strings_2[2]; @@ -16,4 +19,8 @@ template std::string concatStringsSep(std::string_view, const strings_3 &); typedef std::string_view strings_4[4]; template std::string concatStringsSep(std::string_view, const strings_4 &); +template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::list &); +template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::set &); +template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::vector &); + } // namespace nix diff --git a/src/libutil/strings.hh b/src/libutil/strings.hh index 3b112c409..6e991e490 100644 --- a/src/libutil/strings.hh +++ b/src/libutil/strings.hh @@ -8,6 +8,18 @@ namespace nix { +/** + * String tokenizer. + * + * See also `basicSplitString()`, which preserves empty strings between separators, as well as at the start and end. + */ +template +C tokenizeString(std::string_view s, std::string_view separators = " \t\n\r"); + +extern template std::list tokenizeString(std::string_view s, std::string_view separators); +extern template std::set tokenizeString(std::string_view s, std::string_view separators); +extern template std::vector tokenizeString(std::string_view s, std::string_view separators); + /** * Concatenate the given strings with a separator between the elements. */ @@ -18,4 +30,20 @@ extern template std::string concatStringsSep(std::string_view, const std::list &); extern template std::string concatStringsSep(std::string_view, const std::vector &); +/** + * Ignore any empty strings at the start of the list, and then concatenate the + * given strings with a separator between the elements. + * + * @deprecated This function exists for historical reasons. You probably just + * want to use `concatStringsSep`. + */ +template +[[deprecated( + "Consider removing the empty string dropping behavior. If acceptable, use concatStringsSep instead.")]] std::string +dropEmptyInitThenConcatStringsSep(const std::string_view sep, const C & ss); + +extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::list &); +extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::set &); +extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::vector &); + } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 698e181a1..460802be3 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -53,24 +53,6 @@ std::vector stringsToCharPtrs(const Strings & ss) ////////////////////////////////////////////////////////////////////// -template C tokenizeString(std::string_view s, std::string_view separators) -{ - C result; - auto pos = s.find_first_not_of(separators, 0); - while (pos != s.npos) { - auto end = s.find_first_of(separators, pos + 1); - if (end == s.npos) end = s.size(); - result.insert(result.end(), std::string(s, pos, end - pos)); - pos = s.find_first_not_of(separators, end); - } - return result; -} - -template Strings tokenizeString(std::string_view s, std::string_view separators); -template StringSet tokenizeString(std::string_view s, std::string_view separators); -template std::vector tokenizeString(std::string_view s, std::string_view separators); - - std::string chomp(std::string_view s) { size_t i = s.find_last_not_of(" \n\r\t"); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 877d15279..25128a900 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -28,49 +28,11 @@ std::vector stringsToCharPtrs(const Strings & ss); MakeError(FormatError, Error); -/** - * String tokenizer. - */ -template C tokenizeString(std::string_view s, std::string_view separators = " \t\n\r"); - - -/** - * Ignore any empty strings at the start of the list, and then concatenate the - * given strings with a separator between the elements. - * - * @deprecated This function exists for historical reasons. You probably just - * want to use `concatStringsSep`. - */ -template -[[deprecated("Consider removing the empty string dropping behavior. If acceptable, use concatStringsSep instead.")]] -std::string dropEmptyInitThenConcatStringsSep(const std::string_view sep, const C & ss) -{ - size_t size = 0; - - // TODO? remove to make sure we don't rely on the empty item ignoring behavior, - // or just get rid of this function by understanding the remaining calls. - // for (auto & i : ss) { - // // Make sure we don't rely on the empty item ignoring behavior - // assert(!i.empty()); - // break; - // } - - // need a cast to string_view since this is also called with Symbols - for (const auto & s : ss) size += sep.size() + std::string_view(s).size(); - std::string s; - s.reserve(size); - for (auto & i : ss) { - if (s.size() != 0) s += sep; - s += i; - } - return s; -} - -template -auto concatStrings(Parts && ... parts) +template +auto concatStrings(Parts &&... parts) -> std::enable_if_t<(... && std::is_convertible_v), std::string> { - std::string_view views[sizeof...(parts)] = { parts... }; + std::string_view views[sizeof...(parts)] = {parts...}; return concatStringsSep({}, views); } diff --git a/tests/unit/libutil/file-system.cc b/tests/unit/libutil/file-system.cc new file mode 100644 index 000000000..cfddaae1c --- /dev/null +++ b/tests/unit/libutil/file-system.cc @@ -0,0 +1,258 @@ +#include "util.hh" +#include "types.hh" +#include "file-system.hh" +#include "processes.hh" +#include "terminal.hh" +#include "strings.hh" + +#include +#include +#include + +#include + +#ifdef _WIN32 +# define FS_SEP "\\" +# define FS_ROOT "C:" FS_SEP // Need a mounted one, C drive is likely +#else +# define FS_SEP "/" +# define FS_ROOT FS_SEP +#endif + +#ifndef PATH_MAX +# define PATH_MAX 4096 +#endif + +namespace nix { + +/* ----------- tests for file-system.hh -------------------------------------*/ + +/* ---------------------------------------------------------------------------- + * absPath + * --------------------------------------------------------------------------*/ + +TEST(absPath, doesntChangeRoot) +{ + auto p = absPath(FS_ROOT); + + ASSERT_EQ(p, FS_ROOT); +} + +TEST(absPath, turnsEmptyPathIntoCWD) +{ + char cwd[PATH_MAX + 1]; + auto p = absPath(""); + + ASSERT_EQ(p, getcwd((char *) &cwd, PATH_MAX)); +} + +TEST(absPath, usesOptionalBasePathWhenGiven) +{ + char _cwd[PATH_MAX + 1]; + char * cwd = getcwd((char *) &_cwd, PATH_MAX); + + auto p = absPath("", cwd); + + ASSERT_EQ(p, cwd); +} + +TEST(absPath, isIdempotent) +{ + char _cwd[PATH_MAX + 1]; + char * cwd = getcwd((char *) &_cwd, PATH_MAX); + auto p1 = absPath(cwd); + auto p2 = absPath(p1); + + ASSERT_EQ(p1, p2); +} + +TEST(absPath, pathIsCanonicalised) +{ + auto path = FS_ROOT "some/path/with/trailing/dot/."; + auto p1 = absPath(path); + auto p2 = absPath(p1); + + ASSERT_EQ(p1, FS_ROOT "some" FS_SEP "path" FS_SEP "with" FS_SEP "trailing" FS_SEP "dot"); + ASSERT_EQ(p1, p2); +} + +/* ---------------------------------------------------------------------------- + * canonPath + * --------------------------------------------------------------------------*/ + +TEST(canonPath, removesTrailingSlashes) +{ + auto path = FS_ROOT "this/is/a/path//"; + auto p = canonPath(path); + + ASSERT_EQ(p, FS_ROOT "this" FS_SEP "is" FS_SEP "a" FS_SEP "path"); +} + +TEST(canonPath, removesDots) +{ + auto path = FS_ROOT "this/./is/a/path/./"; + auto p = canonPath(path); + + ASSERT_EQ(p, FS_ROOT "this" FS_SEP "is" FS_SEP "a" FS_SEP "path"); +} + +TEST(canonPath, removesDots2) +{ + auto path = FS_ROOT "this/a/../is/a////path/foo/.."; + auto p = canonPath(path); + + ASSERT_EQ(p, FS_ROOT "this" FS_SEP "is" FS_SEP "a" FS_SEP "path"); +} + +TEST(canonPath, requiresAbsolutePath) +{ + ASSERT_ANY_THROW(canonPath(".")); + ASSERT_ANY_THROW(canonPath("..")); + ASSERT_ANY_THROW(canonPath("../")); + ASSERT_DEATH({ canonPath(""); }, "path != \"\""); +} + +/* ---------------------------------------------------------------------------- + * dirOf + * --------------------------------------------------------------------------*/ + +TEST(dirOf, returnsEmptyStringForRoot) +{ + auto p = dirOf("/"); + + ASSERT_EQ(p, "/"); +} + +TEST(dirOf, returnsFirstPathComponent) +{ + auto p1 = dirOf("/dir/"); + ASSERT_EQ(p1, "/dir"); + auto p2 = dirOf("/dir"); + ASSERT_EQ(p2, "/"); + auto p3 = dirOf("/dir/.."); + ASSERT_EQ(p3, "/dir"); + auto p4 = dirOf("/dir/../"); + ASSERT_EQ(p4, "/dir/.."); +} + +/* ---------------------------------------------------------------------------- + * baseNameOf + * --------------------------------------------------------------------------*/ + +TEST(baseNameOf, emptyPath) +{ + auto p1 = baseNameOf(""); + ASSERT_EQ(p1, ""); +} + +TEST(baseNameOf, pathOnRoot) +{ + auto p1 = baseNameOf("/dir"); + ASSERT_EQ(p1, "dir"); +} + +TEST(baseNameOf, relativePath) +{ + auto p1 = baseNameOf("dir/foo"); + ASSERT_EQ(p1, "foo"); +} + +TEST(baseNameOf, pathWithTrailingSlashRoot) +{ + auto p1 = baseNameOf("/"); + ASSERT_EQ(p1, ""); +} + +TEST(baseNameOf, trailingSlash) +{ + auto p1 = baseNameOf("/dir/"); + ASSERT_EQ(p1, "dir"); +} + +TEST(baseNameOf, trailingSlashes) +{ + auto p1 = baseNameOf("/dir//"); + ASSERT_EQ(p1, "dir"); +} + +TEST(baseNameOf, absoluteNothingSlashNothing) +{ + auto p1 = baseNameOf("//"); + ASSERT_EQ(p1, ""); +} + +/* ---------------------------------------------------------------------------- + * isInDir + * --------------------------------------------------------------------------*/ + +TEST(isInDir, trivialCase) +{ + auto p1 = isInDir("/foo/bar", "/foo"); + ASSERT_EQ(p1, true); +} + +TEST(isInDir, notInDir) +{ + auto p1 = isInDir("/zes/foo/bar", "/foo"); + ASSERT_EQ(p1, false); +} + +// XXX: hm, bug or feature? :) Looking at the implementation +// this might be problematic. +TEST(isInDir, emptyDir) +{ + auto p1 = isInDir("/zes/foo/bar", ""); + ASSERT_EQ(p1, true); +} + +/* ---------------------------------------------------------------------------- + * isDirOrInDir + * --------------------------------------------------------------------------*/ + +TEST(isDirOrInDir, trueForSameDirectory) +{ + ASSERT_EQ(isDirOrInDir("/nix", "/nix"), true); + ASSERT_EQ(isDirOrInDir("/", "/"), true); +} + +TEST(isDirOrInDir, trueForEmptyPaths) +{ + ASSERT_EQ(isDirOrInDir("", ""), true); +} + +TEST(isDirOrInDir, falseForDisjunctPaths) +{ + ASSERT_EQ(isDirOrInDir("/foo", "/bar"), false); +} + +TEST(isDirOrInDir, relativePaths) +{ + ASSERT_EQ(isDirOrInDir("/foo/..", "/foo"), true); +} + +// XXX: while it is possible to use "." or ".." in the +// first argument this doesn't seem to work in the second. +TEST(isDirOrInDir, DISABLED_shouldWork) +{ + ASSERT_EQ(isDirOrInDir("/foo/..", "/foo/."), true); +} + +/* ---------------------------------------------------------------------------- + * pathExists + * --------------------------------------------------------------------------*/ + +TEST(pathExists, rootExists) +{ + ASSERT_TRUE(pathExists(FS_ROOT)); +} + +TEST(pathExists, cwdExists) +{ + ASSERT_TRUE(pathExists(".")); +} + +TEST(pathExists, bogusPathDoesNotExist) +{ + ASSERT_FALSE(pathExists("/schnitzel/darmstadt/pommes")); +} +} diff --git a/tests/unit/libutil/meson.build b/tests/unit/libutil/meson.build index 7f024e6f2..f4f2ae7f9 100644 --- a/tests/unit/libutil/meson.build +++ b/tests/unit/libutil/meson.build @@ -61,11 +61,14 @@ sources = files( 'lru-cache.cc', 'nix_api_util.cc', 'pool.cc', + 'processes.cc', 'references.cc', 'spawn.cc', + 'strings.cc', 'suggestions.cc', - 'tests.cc', + 'terminal.cc', 'url.cc', + 'util.cc', 'xml-writer.cc', ) diff --git a/tests/unit/libutil/processes.cc b/tests/unit/libutil/processes.cc new file mode 100644 index 000000000..9033595e8 --- /dev/null +++ b/tests/unit/libutil/processes.cc @@ -0,0 +1,17 @@ +#include "processes.hh" + +#include + +namespace nix { + +/* ---------------------------------------------------------------------------- + * statusOk + * --------------------------------------------------------------------------*/ + +TEST(statusOk, zeroIsOk) +{ + ASSERT_EQ(statusOk(0), true); + ASSERT_EQ(statusOk(1), false); +} + +} // namespace nix diff --git a/tests/unit/libutil/strings.cc b/tests/unit/libutil/strings.cc index 47a20770e..0bd2fe0a5 100644 --- a/tests/unit/libutil/strings.cc +++ b/tests/unit/libutil/strings.cc @@ -80,4 +80,155 @@ TEST(concatStringsSep, buildSingleString) ASSERT_EQ(concatStringsSep(",", strings), "this"); } +/* ---------------------------------------------------------------------------- + * dropEmptyInitThenConcatStringsSep + * --------------------------------------------------------------------------*/ + +TEST(dropEmptyInitThenConcatStringsSep, empty) +{ + Strings strings; + + ASSERT_EQ(dropEmptyInitThenConcatStringsSep(",", strings), ""); +} + +TEST(dropEmptyInitThenConcatStringsSep, buildCommaSeparatedString) +{ + Strings strings; + strings.push_back("this"); + strings.push_back("is"); + strings.push_back("great"); + + ASSERT_EQ(dropEmptyInitThenConcatStringsSep(",", strings), "this,is,great"); +} + +TEST(dropEmptyInitThenConcatStringsSep, buildStringWithEmptySeparator) +{ + Strings strings; + strings.push_back("this"); + strings.push_back("is"); + strings.push_back("great"); + + ASSERT_EQ(dropEmptyInitThenConcatStringsSep("", strings), "thisisgreat"); +} + +TEST(dropEmptyInitThenConcatStringsSep, buildSingleString) +{ + Strings strings; + strings.push_back("this"); + strings.push_back(""); + + ASSERT_EQ(dropEmptyInitThenConcatStringsSep(",", strings), "this,"); +} + +TEST(dropEmptyInitThenConcatStringsSep, emptyStrings) +{ + Strings strings; + strings.push_back(""); + strings.push_back(""); + + ASSERT_EQ(dropEmptyInitThenConcatStringsSep(",", strings), ""); +} + +/* ---------------------------------------------------------------------------- + * tokenizeString + * --------------------------------------------------------------------------*/ + +TEST(tokenizeString, empty) +{ + Strings expected = {}; + + ASSERT_EQ(tokenizeString(""), expected); +} + +TEST(tokenizeString, oneSep) +{ + Strings expected = {}; + + ASSERT_EQ(tokenizeString(" "), expected); +} + +TEST(tokenizeString, twoSep) +{ + Strings expected = {}; + + ASSERT_EQ(tokenizeString(" \n"), expected); +} + +TEST(tokenizeString, tokenizeSpacesWithDefaults) +{ + auto s = "foo bar baz"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s), expected); +} + +TEST(tokenizeString, tokenizeTabsWithDefaults) +{ + auto s = "foo\tbar\tbaz"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s), expected); +} + +TEST(tokenizeString, tokenizeTabsSpacesWithDefaults) +{ + auto s = "foo\t bar\t baz"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s), expected); +} + +TEST(tokenizeString, tokenizeTabsSpacesNewlineWithDefaults) +{ + auto s = "foo\t\n bar\t\n baz"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s), expected); +} + +TEST(tokenizeString, tokenizeTabsSpacesNewlineRetWithDefaults) +{ + auto s = "foo\t\n\r bar\t\n\r baz"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s), expected); + + auto s2 = "foo \t\n\r bar \t\n\r baz"; + Strings expected2 = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s2), expected2); +} + +TEST(tokenizeString, tokenizeWithCustomSep) +{ + auto s = "foo\n,bar\n,baz\n"; + Strings expected = {"foo\n", "bar\n", "baz\n"}; + + ASSERT_EQ(tokenizeString(s, ","), expected); +} + +TEST(tokenizeString, tokenizeSepAtStart) +{ + auto s = ",foo,bar,baz"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s, ","), expected); +} + +TEST(tokenizeString, tokenizeSepAtEnd) +{ + auto s = "foo,bar,baz,"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s, ","), expected); +} + +TEST(tokenizeString, tokenizeSepEmpty) +{ + auto s = "foo,,baz"; + Strings expected = {"foo", "baz"}; + + ASSERT_EQ(tokenizeString(s, ","), expected); +} + } // namespace nix diff --git a/tests/unit/libutil/terminal.cc b/tests/unit/libutil/terminal.cc new file mode 100644 index 000000000..cdeb9fd94 --- /dev/null +++ b/tests/unit/libutil/terminal.cc @@ -0,0 +1,60 @@ +#include "util.hh" +#include "types.hh" +#include "terminal.hh" +#include "strings.hh" + +#include +#include + +#include + +namespace nix { + +/* ---------------------------------------------------------------------------- + * filterANSIEscapes + * --------------------------------------------------------------------------*/ + +TEST(filterANSIEscapes, emptyString) +{ + auto s = ""; + auto expected = ""; + + ASSERT_EQ(filterANSIEscapes(s), expected); +} + +TEST(filterANSIEscapes, doesntChangePrintableChars) +{ + auto s = "09 2q304ruyhr slk2-19024 kjsadh sar f"; + + ASSERT_EQ(filterANSIEscapes(s), s); +} + +TEST(filterANSIEscapes, filtersColorCodes) +{ + auto s = "\u001b[30m A \u001b[31m B \u001b[32m C \u001b[33m D \u001b[0m"; + + ASSERT_EQ(filterANSIEscapes(s, true, 2), " A"); + ASSERT_EQ(filterANSIEscapes(s, true, 3), " A "); + ASSERT_EQ(filterANSIEscapes(s, true, 4), " A "); + ASSERT_EQ(filterANSIEscapes(s, true, 5), " A B"); + ASSERT_EQ(filterANSIEscapes(s, true, 8), " A B C"); +} + +TEST(filterANSIEscapes, expandsTabs) +{ + auto s = "foo\tbar\tbaz"; + + ASSERT_EQ(filterANSIEscapes(s, true), "foo bar baz"); +} + +TEST(filterANSIEscapes, utf8) +{ + ASSERT_EQ(filterANSIEscapes("foobar", true, 5), "fooba"); + ASSERT_EQ(filterANSIEscapes("fóóbär", true, 6), "fóóbär"); + ASSERT_EQ(filterANSIEscapes("fóóbär", true, 5), "fóóbä"); + ASSERT_EQ(filterANSIEscapes("fóóbär", true, 3), "fóó"); + ASSERT_EQ(filterANSIEscapes("f€€bär", true, 4), "f€€b"); + ASSERT_EQ(filterANSIEscapes("f𐍈𐍈bär", true, 4), "f𐍈𐍈b"); +} + +} // namespace nix diff --git a/tests/unit/libutil/tests.cc b/tests/unit/libutil/tests.cc deleted file mode 100644 index 2b73d323b..000000000 --- a/tests/unit/libutil/tests.cc +++ /dev/null @@ -1,703 +0,0 @@ -#include "util.hh" -#include "types.hh" -#include "file-system.hh" -#include "processes.hh" -#include "terminal.hh" - -#include -#include - -#include - -#ifdef _WIN32 -# define FS_SEP "\\" -# define FS_ROOT "C:" FS_SEP // Need a mounted one, C drive is likely -#else -# define FS_SEP "/" -# define FS_ROOT FS_SEP -#endif - -#ifndef PATH_MAX -# define PATH_MAX 4096 -#endif - -namespace nix { - -/* ----------- tests for util.hh ------------------------------------------------*/ - - /* ---------------------------------------------------------------------------- - * absPath - * --------------------------------------------------------------------------*/ - - TEST(absPath, doesntChangeRoot) { - auto p = absPath(FS_ROOT); - - ASSERT_EQ(p, FS_ROOT); - } - - - - - TEST(absPath, turnsEmptyPathIntoCWD) { - char cwd[PATH_MAX+1]; - auto p = absPath(""); - - ASSERT_EQ(p, getcwd((char*)&cwd, PATH_MAX)); - } - - TEST(absPath, usesOptionalBasePathWhenGiven) { - char _cwd[PATH_MAX+1]; - char* cwd = getcwd((char*)&_cwd, PATH_MAX); - - auto p = absPath("", cwd); - - ASSERT_EQ(p, cwd); - } - - TEST(absPath, isIdempotent) { - char _cwd[PATH_MAX+1]; - char* cwd = getcwd((char*)&_cwd, PATH_MAX); - auto p1 = absPath(cwd); - auto p2 = absPath(p1); - - ASSERT_EQ(p1, p2); - } - - - TEST(absPath, pathIsCanonicalised) { - auto path = FS_ROOT "some/path/with/trailing/dot/."; - auto p1 = absPath(path); - auto p2 = absPath(p1); - - ASSERT_EQ(p1, FS_ROOT "some" FS_SEP "path" FS_SEP "with" FS_SEP "trailing" FS_SEP "dot"); - ASSERT_EQ(p1, p2); - } - - /* ---------------------------------------------------------------------------- - * canonPath - * --------------------------------------------------------------------------*/ - - TEST(canonPath, removesTrailingSlashes) { - auto path = FS_ROOT "this/is/a/path//"; - auto p = canonPath(path); - - ASSERT_EQ(p, FS_ROOT "this" FS_SEP "is" FS_SEP "a" FS_SEP "path"); - } - - TEST(canonPath, removesDots) { - auto path = FS_ROOT "this/./is/a/path/./"; - auto p = canonPath(path); - - ASSERT_EQ(p, FS_ROOT "this" FS_SEP "is" FS_SEP "a" FS_SEP "path"); - } - - TEST(canonPath, removesDots2) { - auto path = FS_ROOT "this/a/../is/a////path/foo/.."; - auto p = canonPath(path); - - ASSERT_EQ(p, FS_ROOT "this" FS_SEP "is" FS_SEP "a" FS_SEP "path"); - } - - TEST(canonPath, requiresAbsolutePath) { - ASSERT_ANY_THROW(canonPath(".")); - ASSERT_ANY_THROW(canonPath("..")); - ASSERT_ANY_THROW(canonPath("../")); - ASSERT_DEATH({ canonPath(""); }, "path != \"\""); - } - - /* ---------------------------------------------------------------------------- - * dirOf - * --------------------------------------------------------------------------*/ - - TEST(dirOf, returnsEmptyStringForRoot) { - auto p = dirOf("/"); - - ASSERT_EQ(p, "/"); - } - - TEST(dirOf, returnsFirstPathComponent) { - auto p1 = dirOf("/dir/"); - ASSERT_EQ(p1, "/dir"); - auto p2 = dirOf("/dir"); - ASSERT_EQ(p2, "/"); - auto p3 = dirOf("/dir/.."); - ASSERT_EQ(p3, "/dir"); - auto p4 = dirOf("/dir/../"); - ASSERT_EQ(p4, "/dir/.."); - } - - /* ---------------------------------------------------------------------------- - * baseNameOf - * --------------------------------------------------------------------------*/ - - TEST(baseNameOf, emptyPath) { - auto p1 = baseNameOf(""); - ASSERT_EQ(p1, ""); - } - - TEST(baseNameOf, pathOnRoot) { - auto p1 = baseNameOf("/dir"); - ASSERT_EQ(p1, "dir"); - } - - TEST(baseNameOf, relativePath) { - auto p1 = baseNameOf("dir/foo"); - ASSERT_EQ(p1, "foo"); - } - - TEST(baseNameOf, pathWithTrailingSlashRoot) { - auto p1 = baseNameOf("/"); - ASSERT_EQ(p1, ""); - } - - TEST(baseNameOf, trailingSlash) { - auto p1 = baseNameOf("/dir/"); - ASSERT_EQ(p1, "dir"); - } - - TEST(baseNameOf, trailingSlashes) { - auto p1 = baseNameOf("/dir//"); - ASSERT_EQ(p1, "dir"); - } - - TEST(baseNameOf, absoluteNothingSlashNothing) { - auto p1 = baseNameOf("//"); - ASSERT_EQ(p1, ""); - } - - /* ---------------------------------------------------------------------------- - * isInDir - * --------------------------------------------------------------------------*/ - - TEST(isInDir, trivialCase) { - auto p1 = isInDir("/foo/bar", "/foo"); - ASSERT_EQ(p1, true); - } - - TEST(isInDir, notInDir) { - auto p1 = isInDir("/zes/foo/bar", "/foo"); - ASSERT_EQ(p1, false); - } - - // XXX: hm, bug or feature? :) Looking at the implementation - // this might be problematic. - TEST(isInDir, emptyDir) { - auto p1 = isInDir("/zes/foo/bar", ""); - ASSERT_EQ(p1, true); - } - - /* ---------------------------------------------------------------------------- - * isDirOrInDir - * --------------------------------------------------------------------------*/ - - TEST(isDirOrInDir, trueForSameDirectory) { - ASSERT_EQ(isDirOrInDir("/nix", "/nix"), true); - ASSERT_EQ(isDirOrInDir("/", "/"), true); - } - - TEST(isDirOrInDir, trueForEmptyPaths) { - ASSERT_EQ(isDirOrInDir("", ""), true); - } - - TEST(isDirOrInDir, falseForDisjunctPaths) { - ASSERT_EQ(isDirOrInDir("/foo", "/bar"), false); - } - - TEST(isDirOrInDir, relativePaths) { - ASSERT_EQ(isDirOrInDir("/foo/..", "/foo"), true); - } - - // XXX: while it is possible to use "." or ".." in the - // first argument this doesn't seem to work in the second. - TEST(isDirOrInDir, DISABLED_shouldWork) { - ASSERT_EQ(isDirOrInDir("/foo/..", "/foo/."), true); - - } - - /* ---------------------------------------------------------------------------- - * pathExists - * --------------------------------------------------------------------------*/ - - TEST(pathExists, rootExists) { - ASSERT_TRUE(pathExists(FS_ROOT)); - } - - TEST(pathExists, cwdExists) { - ASSERT_TRUE(pathExists(".")); - } - - TEST(pathExists, bogusPathDoesNotExist) { - ASSERT_FALSE(pathExists("/schnitzel/darmstadt/pommes")); - } - - /* ---------------------------------------------------------------------------- - * dropEmptyInitThenConcatStringsSep - * --------------------------------------------------------------------------*/ - - TEST(dropEmptyInitThenConcatStringsSep, buildCommaSeparatedString) { - Strings strings; - strings.push_back("this"); - strings.push_back("is"); - strings.push_back("great"); - - ASSERT_EQ(dropEmptyInitThenConcatStringsSep(",", strings), "this,is,great"); - } - - TEST(dropEmptyInitThenConcatStringsSep, buildStringWithEmptySeparator) { - Strings strings; - strings.push_back("this"); - strings.push_back("is"); - strings.push_back("great"); - - ASSERT_EQ(dropEmptyInitThenConcatStringsSep("", strings), "thisisgreat"); - } - - TEST(dropEmptyInitThenConcatStringsSep, buildSingleString) { - Strings strings; - strings.push_back("this"); - - ASSERT_EQ(dropEmptyInitThenConcatStringsSep(",", strings), "this"); - } - - /* ---------------------------------------------------------------------------- - * hasPrefix - * --------------------------------------------------------------------------*/ - - TEST(hasPrefix, emptyStringHasNoPrefix) { - ASSERT_FALSE(hasPrefix("", "foo")); - } - - TEST(hasPrefix, emptyStringIsAlwaysPrefix) { - ASSERT_TRUE(hasPrefix("foo", "")); - ASSERT_TRUE(hasPrefix("jshjkfhsadf", "")); - } - - TEST(hasPrefix, trivialCase) { - ASSERT_TRUE(hasPrefix("foobar", "foo")); - } - - /* ---------------------------------------------------------------------------- - * hasSuffix - * --------------------------------------------------------------------------*/ - - TEST(hasSuffix, emptyStringHasNoSuffix) { - ASSERT_FALSE(hasSuffix("", "foo")); - } - - TEST(hasSuffix, trivialCase) { - ASSERT_TRUE(hasSuffix("foo", "foo")); - ASSERT_TRUE(hasSuffix("foobar", "bar")); - } - - /* ---------------------------------------------------------------------------- - * base64Encode - * --------------------------------------------------------------------------*/ - - TEST(base64Encode, emptyString) { - ASSERT_EQ(base64Encode(""), ""); - } - - TEST(base64Encode, encodesAString) { - ASSERT_EQ(base64Encode("quod erat demonstrandum"), "cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="); - } - - TEST(base64Encode, encodeAndDecode) { - auto s = "quod erat demonstrandum"; - auto encoded = base64Encode(s); - auto decoded = base64Decode(encoded); - - ASSERT_EQ(decoded, s); - } - - TEST(base64Encode, encodeAndDecodeNonPrintable) { - char s[256]; - std::iota(std::rbegin(s), std::rend(s), 0); - - auto encoded = base64Encode(s); - auto decoded = base64Decode(encoded); - - EXPECT_EQ(decoded.length(), 255); - ASSERT_EQ(decoded, s); - } - - /* ---------------------------------------------------------------------------- - * base64Decode - * --------------------------------------------------------------------------*/ - - TEST(base64Decode, emptyString) { - ASSERT_EQ(base64Decode(""), ""); - } - - TEST(base64Decode, decodeAString) { - ASSERT_EQ(base64Decode("cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="), "quod erat demonstrandum"); - } - - TEST(base64Decode, decodeThrowsOnInvalidChar) { - ASSERT_THROW(base64Decode("cXVvZCBlcm_0IGRlbW9uc3RyYW5kdW0="), Error); - } - - /* ---------------------------------------------------------------------------- - * getLine - * --------------------------------------------------------------------------*/ - - TEST(getLine, all) { - { - auto [line, rest] = getLine("foo\nbar\nxyzzy"); - ASSERT_EQ(line, "foo"); - ASSERT_EQ(rest, "bar\nxyzzy"); - } - - { - auto [line, rest] = getLine("foo\r\nbar\r\nxyzzy"); - ASSERT_EQ(line, "foo"); - ASSERT_EQ(rest, "bar\r\nxyzzy"); - } - - { - auto [line, rest] = getLine("foo\n"); - ASSERT_EQ(line, "foo"); - ASSERT_EQ(rest, ""); - } - - { - auto [line, rest] = getLine("foo"); - ASSERT_EQ(line, "foo"); - ASSERT_EQ(rest, ""); - } - - { - auto [line, rest] = getLine(""); - ASSERT_EQ(line, ""); - ASSERT_EQ(rest, ""); - } - } - - /* ---------------------------------------------------------------------------- - * toLower - * --------------------------------------------------------------------------*/ - - TEST(toLower, emptyString) { - ASSERT_EQ(toLower(""), ""); - } - - TEST(toLower, nonLetters) { - auto s = "!@(*$#)(@#=\\234_"; - ASSERT_EQ(toLower(s), s); - } - - // std::tolower() doesn't handle unicode characters. In the context of - // store paths this isn't relevant but doesn't hurt to record this behavior - // here. - TEST(toLower, umlauts) { - auto s = "ÄÖÜ"; - ASSERT_EQ(toLower(s), "ÄÖÜ"); - } - - /* ---------------------------------------------------------------------------- - * string2Float - * --------------------------------------------------------------------------*/ - - TEST(string2Float, emptyString) { - ASSERT_EQ(string2Float(""), std::nullopt); - } - - TEST(string2Float, trivialConversions) { - ASSERT_EQ(string2Float("1.0"), 1.0); - - ASSERT_EQ(string2Float("0.0"), 0.0); - - ASSERT_EQ(string2Float("-100.25"), -100.25); - } - - /* ---------------------------------------------------------------------------- - * string2Int - * --------------------------------------------------------------------------*/ - - TEST(string2Int, emptyString) { - ASSERT_EQ(string2Int(""), std::nullopt); - } - - TEST(string2Int, trivialConversions) { - ASSERT_EQ(string2Int("1"), 1); - - ASSERT_EQ(string2Int("0"), 0); - - ASSERT_EQ(string2Int("-100"), -100); - } - - /* ---------------------------------------------------------------------------- - * renderSize - * --------------------------------------------------------------------------*/ - - TEST(renderSize, misc) { - ASSERT_EQ(renderSize(0, true), " 0.0 KiB"); - ASSERT_EQ(renderSize(100, true), " 0.1 KiB"); - ASSERT_EQ(renderSize(100), "0.1 KiB"); - ASSERT_EQ(renderSize(972, true), " 0.9 KiB"); - ASSERT_EQ(renderSize(973, true), " 1.0 KiB"); // FIXME: should round down - ASSERT_EQ(renderSize(1024, true), " 1.0 KiB"); - ASSERT_EQ(renderSize(1024 * 1024, true), "1024.0 KiB"); - ASSERT_EQ(renderSize(1100 * 1024, true), " 1.1 MiB"); - ASSERT_EQ(renderSize(2ULL * 1024 * 1024 * 1024, true), " 2.0 GiB"); - ASSERT_EQ(renderSize(2100ULL * 1024 * 1024 * 1024, true), " 2.1 TiB"); - } - -#ifndef _WIN32 // TODO re-enable on Windows, once we can start processes - /* ---------------------------------------------------------------------------- - * statusOk - * --------------------------------------------------------------------------*/ - - TEST(statusOk, zeroIsOk) { - ASSERT_EQ(statusOk(0), true); - ASSERT_EQ(statusOk(1), false); - } -#endif - - - /* ---------------------------------------------------------------------------- - * rewriteStrings - * --------------------------------------------------------------------------*/ - - TEST(rewriteStrings, emptyString) { - StringMap rewrites; - rewrites["this"] = "that"; - - ASSERT_EQ(rewriteStrings("", rewrites), ""); - } - - TEST(rewriteStrings, emptyRewrites) { - StringMap rewrites; - - ASSERT_EQ(rewriteStrings("this and that", rewrites), "this and that"); - } - - TEST(rewriteStrings, successfulRewrite) { - StringMap rewrites; - rewrites["this"] = "that"; - - ASSERT_EQ(rewriteStrings("this and that", rewrites), "that and that"); - } - - TEST(rewriteStrings, doesntOccur) { - StringMap rewrites; - rewrites["foo"] = "bar"; - - ASSERT_EQ(rewriteStrings("this and that", rewrites), "this and that"); - } - - /* ---------------------------------------------------------------------------- - * replaceStrings - * --------------------------------------------------------------------------*/ - - TEST(replaceStrings, emptyString) { - ASSERT_EQ(replaceStrings("", "this", "that"), ""); - ASSERT_EQ(replaceStrings("this and that", "", ""), "this and that"); - } - - TEST(replaceStrings, successfulReplace) { - ASSERT_EQ(replaceStrings("this and that", "this", "that"), "that and that"); - } - - TEST(replaceStrings, doesntOccur) { - ASSERT_EQ(replaceStrings("this and that", "foo", "bar"), "this and that"); - } - - /* ---------------------------------------------------------------------------- - * trim - * --------------------------------------------------------------------------*/ - - TEST(trim, emptyString) { - ASSERT_EQ(trim(""), ""); - } - - TEST(trim, removesWhitespace) { - ASSERT_EQ(trim("foo"), "foo"); - ASSERT_EQ(trim(" foo "), "foo"); - ASSERT_EQ(trim(" foo bar baz"), "foo bar baz"); - ASSERT_EQ(trim(" \t foo bar baz\n"), "foo bar baz"); - } - - /* ---------------------------------------------------------------------------- - * chomp - * --------------------------------------------------------------------------*/ - - TEST(chomp, emptyString) { - ASSERT_EQ(chomp(""), ""); - } - - TEST(chomp, removesWhitespace) { - ASSERT_EQ(chomp("foo"), "foo"); - ASSERT_EQ(chomp("foo "), "foo"); - ASSERT_EQ(chomp(" foo "), " foo"); - ASSERT_EQ(chomp(" foo bar baz "), " foo bar baz"); - ASSERT_EQ(chomp("\t foo bar baz\n"), "\t foo bar baz"); - } - - /* ---------------------------------------------------------------------------- - * quoteStrings - * --------------------------------------------------------------------------*/ - - TEST(quoteStrings, empty) { - Strings s = { }; - Strings expected = { }; - - ASSERT_EQ(quoteStrings(s), expected); - } - - TEST(quoteStrings, emptyStrings) { - Strings s = { "", "", "" }; - Strings expected = { "''", "''", "''" }; - ASSERT_EQ(quoteStrings(s), expected); - - } - - TEST(quoteStrings, trivialQuote) { - Strings s = { "foo", "bar", "baz" }; - Strings expected = { "'foo'", "'bar'", "'baz'" }; - - ASSERT_EQ(quoteStrings(s), expected); - } - - TEST(quoteStrings, quotedStrings) { - Strings s = { "'foo'", "'bar'", "'baz'" }; - Strings expected = { "''foo''", "''bar''", "''baz''" }; - - ASSERT_EQ(quoteStrings(s), expected); - } - - /* ---------------------------------------------------------------------------- - * tokenizeString - * --------------------------------------------------------------------------*/ - - TEST(tokenizeString, empty) { - Strings expected = { }; - - ASSERT_EQ(tokenizeString(""), expected); - } - - TEST(tokenizeString, tokenizeSpacesWithDefaults) { - auto s = "foo bar baz"; - Strings expected = { "foo", "bar", "baz" }; - - ASSERT_EQ(tokenizeString(s), expected); - } - - TEST(tokenizeString, tokenizeTabsWithDefaults) { - auto s = "foo\tbar\tbaz"; - Strings expected = { "foo", "bar", "baz" }; - - ASSERT_EQ(tokenizeString(s), expected); - } - - TEST(tokenizeString, tokenizeTabsSpacesWithDefaults) { - auto s = "foo\t bar\t baz"; - Strings expected = { "foo", "bar", "baz" }; - - ASSERT_EQ(tokenizeString(s), expected); - } - - TEST(tokenizeString, tokenizeTabsSpacesNewlineWithDefaults) { - auto s = "foo\t\n bar\t\n baz"; - Strings expected = { "foo", "bar", "baz" }; - - ASSERT_EQ(tokenizeString(s), expected); - } - - TEST(tokenizeString, tokenizeTabsSpacesNewlineRetWithDefaults) { - auto s = "foo\t\n\r bar\t\n\r baz"; - Strings expected = { "foo", "bar", "baz" }; - - ASSERT_EQ(tokenizeString(s), expected); - - auto s2 = "foo \t\n\r bar \t\n\r baz"; - Strings expected2 = { "foo", "bar", "baz" }; - - ASSERT_EQ(tokenizeString(s2), expected2); - } - - TEST(tokenizeString, tokenizeWithCustomSep) { - auto s = "foo\n,bar\n,baz\n"; - Strings expected = { "foo\n", "bar\n", "baz\n" }; - - ASSERT_EQ(tokenizeString(s, ","), expected); - } - - /* ---------------------------------------------------------------------------- - * get - * --------------------------------------------------------------------------*/ - - TEST(get, emptyContainer) { - StringMap s = { }; - auto expected = nullptr; - - ASSERT_EQ(get(s, "one"), expected); - } - - TEST(get, getFromContainer) { - StringMap s; - s["one"] = "yi"; - s["two"] = "er"; - auto expected = "yi"; - - ASSERT_EQ(*get(s, "one"), expected); - } - - TEST(getOr, emptyContainer) { - StringMap s = { }; - auto expected = "yi"; - - ASSERT_EQ(getOr(s, "one", "yi"), expected); - } - - TEST(getOr, getFromContainer) { - StringMap s; - s["one"] = "yi"; - s["two"] = "er"; - auto expected = "yi"; - - ASSERT_EQ(getOr(s, "one", "nope"), expected); - } - - /* ---------------------------------------------------------------------------- - * filterANSIEscapes - * --------------------------------------------------------------------------*/ - - TEST(filterANSIEscapes, emptyString) { - auto s = ""; - auto expected = ""; - - ASSERT_EQ(filterANSIEscapes(s), expected); - } - - TEST(filterANSIEscapes, doesntChangePrintableChars) { - auto s = "09 2q304ruyhr slk2-19024 kjsadh sar f"; - - ASSERT_EQ(filterANSIEscapes(s), s); - } - - TEST(filterANSIEscapes, filtersColorCodes) { - auto s = "\u001b[30m A \u001b[31m B \u001b[32m C \u001b[33m D \u001b[0m"; - - ASSERT_EQ(filterANSIEscapes(s, true, 2), " A" ); - ASSERT_EQ(filterANSIEscapes(s, true, 3), " A " ); - ASSERT_EQ(filterANSIEscapes(s, true, 4), " A " ); - ASSERT_EQ(filterANSIEscapes(s, true, 5), " A B" ); - ASSERT_EQ(filterANSIEscapes(s, true, 8), " A B C" ); - } - - TEST(filterANSIEscapes, expandsTabs) { - auto s = "foo\tbar\tbaz"; - - ASSERT_EQ(filterANSIEscapes(s, true), "foo bar baz" ); - } - - TEST(filterANSIEscapes, utf8) { - ASSERT_EQ(filterANSIEscapes("foobar", true, 5), "fooba"); - ASSERT_EQ(filterANSIEscapes("fóóbär", true, 6), "fóóbär"); - ASSERT_EQ(filterANSIEscapes("fóóbär", true, 5), "fóóbä"); - ASSERT_EQ(filterANSIEscapes("fóóbär", true, 3), "fóó"); - ASSERT_EQ(filterANSIEscapes("f€€bär", true, 4), "f€€b"); - ASSERT_EQ(filterANSIEscapes("f𐍈𐍈bär", true, 4), "f𐍈𐍈b"); - } - -} diff --git a/tests/unit/libutil/util.cc b/tests/unit/libutil/util.cc new file mode 100644 index 000000000..a3f7c720a --- /dev/null +++ b/tests/unit/libutil/util.cc @@ -0,0 +1,385 @@ +#include "util.hh" +#include "types.hh" +#include "file-system.hh" +#include "terminal.hh" +#include "strings.hh" + +#include +#include + +#include + +namespace nix { + +/* ----------- tests for util.hh --------------------------------------------*/ + +/* ---------------------------------------------------------------------------- + * hasPrefix + * --------------------------------------------------------------------------*/ + +TEST(hasPrefix, emptyStringHasNoPrefix) +{ + ASSERT_FALSE(hasPrefix("", "foo")); +} + +TEST(hasPrefix, emptyStringIsAlwaysPrefix) +{ + ASSERT_TRUE(hasPrefix("foo", "")); + ASSERT_TRUE(hasPrefix("jshjkfhsadf", "")); +} + +TEST(hasPrefix, trivialCase) +{ + ASSERT_TRUE(hasPrefix("foobar", "foo")); +} + +/* ---------------------------------------------------------------------------- + * hasSuffix + * --------------------------------------------------------------------------*/ + +TEST(hasSuffix, emptyStringHasNoSuffix) +{ + ASSERT_FALSE(hasSuffix("", "foo")); +} + +TEST(hasSuffix, trivialCase) +{ + ASSERT_TRUE(hasSuffix("foo", "foo")); + ASSERT_TRUE(hasSuffix("foobar", "bar")); +} + +/* ---------------------------------------------------------------------------- + * base64Encode + * --------------------------------------------------------------------------*/ + +TEST(base64Encode, emptyString) +{ + ASSERT_EQ(base64Encode(""), ""); +} + +TEST(base64Encode, encodesAString) +{ + ASSERT_EQ(base64Encode("quod erat demonstrandum"), "cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="); +} + +TEST(base64Encode, encodeAndDecode) +{ + auto s = "quod erat demonstrandum"; + auto encoded = base64Encode(s); + auto decoded = base64Decode(encoded); + + ASSERT_EQ(decoded, s); +} + +TEST(base64Encode, encodeAndDecodeNonPrintable) +{ + char s[256]; + std::iota(std::rbegin(s), std::rend(s), 0); + + auto encoded = base64Encode(s); + auto decoded = base64Decode(encoded); + + EXPECT_EQ(decoded.length(), 255); + ASSERT_EQ(decoded, s); +} + +/* ---------------------------------------------------------------------------- + * base64Decode + * --------------------------------------------------------------------------*/ + +TEST(base64Decode, emptyString) +{ + ASSERT_EQ(base64Decode(""), ""); +} + +TEST(base64Decode, decodeAString) +{ + ASSERT_EQ(base64Decode("cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="), "quod erat demonstrandum"); +} + +TEST(base64Decode, decodeThrowsOnInvalidChar) +{ + ASSERT_THROW(base64Decode("cXVvZCBlcm_0IGRlbW9uc3RyYW5kdW0="), Error); +} + +/* ---------------------------------------------------------------------------- + * getLine + * --------------------------------------------------------------------------*/ + +TEST(getLine, all) +{ + { + auto [line, rest] = getLine("foo\nbar\nxyzzy"); + ASSERT_EQ(line, "foo"); + ASSERT_EQ(rest, "bar\nxyzzy"); + } + + { + auto [line, rest] = getLine("foo\r\nbar\r\nxyzzy"); + ASSERT_EQ(line, "foo"); + ASSERT_EQ(rest, "bar\r\nxyzzy"); + } + + { + auto [line, rest] = getLine("foo\n"); + ASSERT_EQ(line, "foo"); + ASSERT_EQ(rest, ""); + } + + { + auto [line, rest] = getLine("foo"); + ASSERT_EQ(line, "foo"); + ASSERT_EQ(rest, ""); + } + + { + auto [line, rest] = getLine(""); + ASSERT_EQ(line, ""); + ASSERT_EQ(rest, ""); + } +} + +/* ---------------------------------------------------------------------------- + * toLower + * --------------------------------------------------------------------------*/ + +TEST(toLower, emptyString) +{ + ASSERT_EQ(toLower(""), ""); +} + +TEST(toLower, nonLetters) +{ + auto s = "!@(*$#)(@#=\\234_"; + ASSERT_EQ(toLower(s), s); +} + +// std::tolower() doesn't handle unicode characters. In the context of +// store paths this isn't relevant but doesn't hurt to record this behavior +// here. +TEST(toLower, umlauts) +{ + auto s = "ÄÖÜ"; + ASSERT_EQ(toLower(s), "ÄÖÜ"); +} + +/* ---------------------------------------------------------------------------- + * string2Float + * --------------------------------------------------------------------------*/ + +TEST(string2Float, emptyString) +{ + ASSERT_EQ(string2Float(""), std::nullopt); +} + +TEST(string2Float, trivialConversions) +{ + ASSERT_EQ(string2Float("1.0"), 1.0); + + ASSERT_EQ(string2Float("0.0"), 0.0); + + ASSERT_EQ(string2Float("-100.25"), -100.25); +} + +/* ---------------------------------------------------------------------------- + * string2Int + * --------------------------------------------------------------------------*/ + +TEST(string2Int, emptyString) +{ + ASSERT_EQ(string2Int(""), std::nullopt); +} + +TEST(string2Int, trivialConversions) +{ + ASSERT_EQ(string2Int("1"), 1); + + ASSERT_EQ(string2Int("0"), 0); + + ASSERT_EQ(string2Int("-100"), -100); +} + +/* ---------------------------------------------------------------------------- + * renderSize + * --------------------------------------------------------------------------*/ + +TEST(renderSize, misc) +{ + ASSERT_EQ(renderSize(0, true), " 0.0 KiB"); + ASSERT_EQ(renderSize(100, true), " 0.1 KiB"); + ASSERT_EQ(renderSize(100), "0.1 KiB"); + ASSERT_EQ(renderSize(972, true), " 0.9 KiB"); + ASSERT_EQ(renderSize(973, true), " 1.0 KiB"); // FIXME: should round down + ASSERT_EQ(renderSize(1024, true), " 1.0 KiB"); + ASSERT_EQ(renderSize(1024 * 1024, true), "1024.0 KiB"); + ASSERT_EQ(renderSize(1100 * 1024, true), " 1.1 MiB"); + ASSERT_EQ(renderSize(2ULL * 1024 * 1024 * 1024, true), " 2.0 GiB"); + ASSERT_EQ(renderSize(2100ULL * 1024 * 1024 * 1024, true), " 2.1 TiB"); +} + +/* ---------------------------------------------------------------------------- + * rewriteStrings + * --------------------------------------------------------------------------*/ + +TEST(rewriteStrings, emptyString) +{ + StringMap rewrites; + rewrites["this"] = "that"; + + ASSERT_EQ(rewriteStrings("", rewrites), ""); +} + +TEST(rewriteStrings, emptyRewrites) +{ + StringMap rewrites; + + ASSERT_EQ(rewriteStrings("this and that", rewrites), "this and that"); +} + +TEST(rewriteStrings, successfulRewrite) +{ + StringMap rewrites; + rewrites["this"] = "that"; + + ASSERT_EQ(rewriteStrings("this and that", rewrites), "that and that"); +} + +TEST(rewriteStrings, doesntOccur) +{ + StringMap rewrites; + rewrites["foo"] = "bar"; + + ASSERT_EQ(rewriteStrings("this and that", rewrites), "this and that"); +} + +/* ---------------------------------------------------------------------------- + * replaceStrings + * --------------------------------------------------------------------------*/ + +TEST(replaceStrings, emptyString) +{ + ASSERT_EQ(replaceStrings("", "this", "that"), ""); + ASSERT_EQ(replaceStrings("this and that", "", ""), "this and that"); +} + +TEST(replaceStrings, successfulReplace) +{ + ASSERT_EQ(replaceStrings("this and that", "this", "that"), "that and that"); +} + +TEST(replaceStrings, doesntOccur) +{ + ASSERT_EQ(replaceStrings("this and that", "foo", "bar"), "this and that"); +} + +/* ---------------------------------------------------------------------------- + * trim + * --------------------------------------------------------------------------*/ + +TEST(trim, emptyString) +{ + ASSERT_EQ(trim(""), ""); +} + +TEST(trim, removesWhitespace) +{ + ASSERT_EQ(trim("foo"), "foo"); + ASSERT_EQ(trim(" foo "), "foo"); + ASSERT_EQ(trim(" foo bar baz"), "foo bar baz"); + ASSERT_EQ(trim(" \t foo bar baz\n"), "foo bar baz"); +} + +/* ---------------------------------------------------------------------------- + * chomp + * --------------------------------------------------------------------------*/ + +TEST(chomp, emptyString) +{ + ASSERT_EQ(chomp(""), ""); +} + +TEST(chomp, removesWhitespace) +{ + ASSERT_EQ(chomp("foo"), "foo"); + ASSERT_EQ(chomp("foo "), "foo"); + ASSERT_EQ(chomp(" foo "), " foo"); + ASSERT_EQ(chomp(" foo bar baz "), " foo bar baz"); + ASSERT_EQ(chomp("\t foo bar baz\n"), "\t foo bar baz"); +} + +/* ---------------------------------------------------------------------------- + * quoteStrings + * --------------------------------------------------------------------------*/ + +TEST(quoteStrings, empty) +{ + Strings s = {}; + Strings expected = {}; + + ASSERT_EQ(quoteStrings(s), expected); +} + +TEST(quoteStrings, emptyStrings) +{ + Strings s = {"", "", ""}; + Strings expected = {"''", "''", "''"}; + ASSERT_EQ(quoteStrings(s), expected); +} + +TEST(quoteStrings, trivialQuote) +{ + Strings s = {"foo", "bar", "baz"}; + Strings expected = {"'foo'", "'bar'", "'baz'"}; + + ASSERT_EQ(quoteStrings(s), expected); +} + +TEST(quoteStrings, quotedStrings) +{ + Strings s = {"'foo'", "'bar'", "'baz'"}; + Strings expected = {"''foo''", "''bar''", "''baz''"}; + + ASSERT_EQ(quoteStrings(s), expected); +} + +/* ---------------------------------------------------------------------------- + * get + * --------------------------------------------------------------------------*/ + +TEST(get, emptyContainer) +{ + StringMap s = {}; + auto expected = nullptr; + + ASSERT_EQ(get(s, "one"), expected); +} + +TEST(get, getFromContainer) +{ + StringMap s; + s["one"] = "yi"; + s["two"] = "er"; + auto expected = "yi"; + + ASSERT_EQ(*get(s, "one"), expected); +} + +TEST(getOr, emptyContainer) +{ + StringMap s = {}; + auto expected = "yi"; + + ASSERT_EQ(getOr(s, "one", "yi"), expected); +} + +TEST(getOr, getFromContainer) +{ + StringMap s; + s["one"] = "yi"; + s["two"] = "er"; + auto expected = "yi"; + + ASSERT_EQ(getOr(s, "one", "nope"), expected); +} + +} // namespace nix From 0a00bd07b2421acfb21751a718292affa8c6e837 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 5 Aug 2024 18:56:02 +0200 Subject: [PATCH 122/142] PathSubstitutionGoal: Fix spurious "failed" count in the progress bar It is not an error if queryPathInfo() indicates that a path does not exist in the substituter. Fixes #11198. This was broken in 846869da0ed0580beb7f827b303fef9a8386de37. --- src/libstore/build/substitution-goal.cc | 8 +++++--- src/libstore/build/substitution-goal.hh | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 7deeb4748..0152f1808 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -145,8 +145,10 @@ Goal::Co PathSubstitutionGoal::init() /* None left. Terminate this goal and let someone else deal with it. */ - worker.failedSubstitutions++; - worker.updateProgress(); + if (substituterFailed) { + worker.failedSubstitutions++; + worker.updateProgress(); + } /* Hack: don't indicate failure if there were no substituters. In that case the calling derivation should just do a @@ -158,7 +160,7 @@ Goal::Co PathSubstitutionGoal::init() } -Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref sub, std::shared_ptr info, bool& substituterFailed) +Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref sub, std::shared_ptr info, bool & substituterFailed) { trace("all references realised"); diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index c1de45379..f2cf797e5 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -66,7 +66,7 @@ public: */ Co init() override; Co gotInfo(); - Co tryToRun(StorePath subPath, nix::ref sub, std::shared_ptr info, bool& substituterFailed); + Co tryToRun(StorePath subPath, nix::ref sub, std::shared_ptr info, bool & substituterFailed); Co finished(); /** From 6c861b9c51b6dece17dbd94e9016d7984159c9a8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 5 Aug 2024 12:05:29 -0400 Subject: [PATCH 123/142] Factor out `lookupExecutable` and other PATH improvments This ended up motivating a good deal of other infra improvements in order to get Windows right: - `OsString` to complement `std::filesystem::path` - env var code for working with the underlying `OsString`s - Rename `PATHNG_LITERAL` to `OS_STR` - `NativePathTrait` renamed to `OsPathTrait`, given a character template parameter until #9205 is complete. Split `tests.cc` matching split of `util.{cc,hh}` last year. Co-authored-by: Robert Hensing --- .clang-format | 1 + maintainers/flake-module.nix | 3 - src/libutil/environment-variables.cc | 11 +- src/libutil/environment-variables.hh | 11 ++ src/libutil/executable-path.cc | 79 +++++++++++++ src/libutil/executable-path.hh | 64 +++++++++++ src/libutil/file-path-impl.hh | 9 +- src/libutil/file-path.hh | 25 +--- src/libutil/file-system.cc | 23 +++- src/libutil/file-system.hh | 6 + src/libutil/meson.build | 3 + src/libutil/os-string.hh | 43 +++++++ src/libutil/strings-inline.hh | 46 +++++++- src/libutil/strings.cc | 9 ++ src/libutil/strings.hh | 20 ++++ src/libutil/unix/environment-variables.cc | 10 ++ src/libutil/unix/file-path.cc | 10 -- src/libutil/unix/meson.build | 1 + src/libutil/unix/os-string.cc | 21 ++++ src/libutil/util.cc | 1 + src/libutil/windows/environment-variables.cc | 30 ++++- src/libutil/windows/file-path.cc | 12 -- src/libutil/windows/meson.build | 1 + src/libutil/windows/os-string.cc | 24 ++++ src/nix/config-check.cc | 36 ++++-- src/nix/develop.cc | 6 +- src/nix/env.cc | 9 +- src/nix/search.cc | 1 + src/nix/upgrade-nix.cc | 19 ++-- tests/unit/libutil/executable-path.cc | 64 +++++++++++ tests/unit/libutil/meson.build | 1 + tests/unit/libutil/strings.cc | 114 +++++++++++++++++++ 32 files changed, 616 insertions(+), 97 deletions(-) create mode 100644 src/libutil/executable-path.cc create mode 100644 src/libutil/executable-path.hh create mode 100644 src/libutil/os-string.hh create mode 100644 src/libutil/unix/os-string.cc create mode 100644 src/libutil/windows/os-string.cc create mode 100644 tests/unit/libutil/executable-path.cc diff --git a/.clang-format b/.clang-format index 3067583e1..4f191fc18 100644 --- a/.clang-format +++ b/.clang-format @@ -31,3 +31,4 @@ AlwaysBreakBeforeMultilineStrings: true IndentPPDirectives: AfterHash PPIndentWidth: 2 BinPackArguments: false +BreakBeforeTernaryOperators: true diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index be91df536..c3eaf671c 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -275,7 +275,6 @@ ''^src/libutil/current-process\.hh$'' ''^src/libutil/english\.cc$'' ''^src/libutil/english\.hh$'' - ''^src/libutil/environment-variables\.cc$'' ''^src/libutil/error\.cc$'' ''^src/libutil/error\.hh$'' ''^src/libutil/exit\.hh$'' @@ -357,7 +356,6 @@ ''^src/libutil/util\.cc$'' ''^src/libutil/util\.hh$'' ''^src/libutil/variant-wrapper\.hh$'' - ''^src/libutil/windows/environment-variables\.cc$'' ''^src/libutil/windows/file-descriptor\.cc$'' ''^src/libutil/windows/file-path\.cc$'' ''^src/libutil/windows/processes\.cc$'' @@ -485,7 +483,6 @@ ''^tests/unit/libutil/pool\.cc'' ''^tests/unit/libutil/references\.cc'' ''^tests/unit/libutil/suggestions\.cc'' - ''^tests/unit/libutil/tests\.cc'' ''^tests/unit/libutil/url\.cc'' ''^tests/unit/libutil/xml-writer\.cc'' ]; diff --git a/src/libutil/environment-variables.cc b/src/libutil/environment-variables.cc index d43197aa0..5947cf742 100644 --- a/src/libutil/environment-variables.cc +++ b/src/libutil/environment-variables.cc @@ -1,20 +1,23 @@ #include "util.hh" #include "environment-variables.hh" -extern char * * environ __attribute__((weak)); +extern char ** environ __attribute__((weak)); namespace nix { std::optional getEnv(const std::string & key) { char * value = getenv(key.c_str()); - if (!value) return {}; + if (!value) + return {}; return std::string(value); } -std::optional getEnvNonEmpty(const std::string & key) { +std::optional getEnvNonEmpty(const std::string & key) +{ auto value = getEnv(key); - if (value == "") return {}; + if (value == "") + return {}; return value; } diff --git a/src/libutil/environment-variables.hh b/src/libutil/environment-variables.hh index e0649adac..879e1f304 100644 --- a/src/libutil/environment-variables.hh +++ b/src/libutil/environment-variables.hh @@ -9,6 +9,7 @@ #include #include "types.hh" +#include "file-path.hh" namespace nix { @@ -17,6 +18,11 @@ namespace nix { */ std::optional getEnv(const std::string & key); +/** + * Like `getEnv`, but using `OsString` to avoid coercions. + */ +std::optional getEnvOs(const OsString & key); + /** * @return a non empty environment variable. Returns nullopt if the env * variable is set to "" @@ -43,6 +49,11 @@ int unsetenv(const char * name); */ int setEnv(const char * name, const char * value); +/** + * Like `setEnv`, but using `OsString` to avoid coercions. + */ +int setEnvOs(const OsString & name, const OsString & value); + /** * Clear the environment. */ diff --git a/src/libutil/executable-path.cc b/src/libutil/executable-path.cc new file mode 100644 index 000000000..1658e3667 --- /dev/null +++ b/src/libutil/executable-path.cc @@ -0,0 +1,79 @@ +#include "environment-variables.hh" +#include "executable-path.hh" +#include "strings-inline.hh" +#include "util.hh" +#include "file-path-impl.hh" + +namespace nix { + +namespace fs = std::filesystem; + +constexpr static const OsStringView path_var_separator{ + &ExecutablePath::separator, + 1, +}; + +ExecutablePath ExecutablePath::load() +{ + // "If PATH is unset or is set to null, the path search is + // implementation-defined." + // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03 + return ExecutablePath::parse(getEnvOs(OS_STR("PATH")).value_or(OS_STR(""))); +} + +ExecutablePath ExecutablePath::parse(const OsString & path) +{ + auto strings = path.empty() ? (std::list{}) + : basicSplitString, OsString::value_type>(path, path_var_separator); + + std::vector ret; + ret.reserve(strings.size()); + + std::transform( + std::make_move_iterator(strings.begin()), + std::make_move_iterator(strings.end()), + std::back_inserter(ret), + [](auto && str) { + return fs::path{ + str.empty() + // "A zero-length prefix is a legacy feature that + // indicates the current working directory. It + // appears as two adjacent characters + // ("::"), as an initial preceding the rest + // of the list, or as a trailing following + // the rest of the list." + // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03 + ? OS_STR(".") + : std::move(str), + }; + }); + + return {ret}; +} + +OsString ExecutablePath::render() const +{ + std::vector path2; + for (auto & p : directories) + path2.push_back(p.native()); + return basicConcatStringsSep(path_var_separator, path2); +} + +std::optional +ExecutablePath::find(const OsString & exe, std::function isExecutable) const +{ + // "If the pathname being sought contains a , the search + // through the path prefixes shall not be performed." + // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03 + assert(OsPathTrait::rfindPathSep(exe) == exe.npos); + + for (auto & dir : directories) { + auto candidate = dir / exe; + if (isExecutable(candidate)) + return std::filesystem::canonical(candidate); + } + + return std::nullopt; +} + +} // namespace nix diff --git a/src/libutil/executable-path.hh b/src/libutil/executable-path.hh new file mode 100644 index 000000000..bcb8d28e8 --- /dev/null +++ b/src/libutil/executable-path.hh @@ -0,0 +1,64 @@ +#pragma once +///@file + +#include "file-system.hh" + +namespace nix { + +struct ExecutablePath +{ + std::vector directories; + + constexpr static const OsString::value_type separator = +#ifdef WIN32 + L';' +#else + ':' +#endif + ; + + /** + * Parse `path` into a list of paths. + * + * On Unix we split on `:`, on Windows we split on `;`. + * + * For Unix, this is according to the POSIX spec for `PATH`. + * https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03 + */ + static ExecutablePath parse(const OsString & path); + + /** + * Load the `PATH` environment variable and `parse` it. + */ + static ExecutablePath load(); + + /** + * Opposite of `parse` + */ + OsString render() const; + + /** + * Search for an executable. + * + * For Unix, this is according to the POSIX spec for `PATH`. + * https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03 + * + * @param exe This must just be a name, and not contain any `/` (or + * `\` on Windows). in case it does, per the spec no lookup should + * be perfomed, and the path (it is not just a file name) as is. + * This is the caller's respsonsibility. + * + * This is a pure function, except for the default `isExecutable` + * argument, which uses the ambient file system to check if a file is + * executable (and exists). + * + * @return path to a resolved executable + */ + std::optional find( + const OsString & exe, + std::function isExecutableFile = isExecutableFileAmbient) const; + + bool operator==(const ExecutablePath &) const = default; +}; + +} // namespace nix diff --git a/src/libutil/file-path-impl.hh b/src/libutil/file-path-impl.hh index 4c90150fd..d7c823fd0 100644 --- a/src/libutil/file-path-impl.hh +++ b/src/libutil/file-path-impl.hh @@ -91,13 +91,10 @@ struct WindowsPathTrait }; -/** - * @todo Revisit choice of `char` or `wchar_t` for `WindowsPathTrait` - * argument. - */ -using NativePathTrait = +template +using OsPathTrait = #ifdef _WIN32 - WindowsPathTrait + WindowsPathTrait #else UnixPathTrait #endif diff --git a/src/libutil/file-path.hh b/src/libutil/file-path.hh index 6589c4060..8e4a88b9d 100644 --- a/src/libutil/file-path.hh +++ b/src/libutil/file-path.hh @@ -1,10 +1,10 @@ #pragma once ///@file -#include #include #include "types.hh" +#include "os-string.hh" namespace nix { @@ -22,39 +22,26 @@ typedef std::set PathSetNG; * * @todo drop `NG` suffix and replace the one in `types.hh`. */ -struct PathViewNG : std::basic_string_view +struct PathViewNG : OsStringView { - using string_view = std::basic_string_view; + using string_view = OsStringView; using string_view::string_view; PathViewNG(const std::filesystem::path & path) - : std::basic_string_view(path.native()) + : OsStringView{path.native()} { } - PathViewNG(const std::filesystem::path::string_type & path) - : std::basic_string_view(path) + PathViewNG(const OsString & path) + : OsStringView{path} { } const string_view & native() const { return *this; } string_view & native() { return *this; } }; -std::string os_string_to_string(PathViewNG::string_view path); - -std::filesystem::path::string_type string_to_os_string(std::string_view s); - std::optional maybePath(PathView path); std::filesystem::path pathNG(PathView path); -/** - * Create string literals with the native character width of paths - */ -#ifndef _WIN32 -# define PATHNG_LITERAL(s) s -#else -# define PATHNG_LITERAL(s) L ## s -#endif - } diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 060a806fb..156dddf98 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -92,7 +92,7 @@ Path canonPath(PathView path, bool resolveSymlinks) arbitrary (but high) limit to prevent infinite loops. */ unsigned int followCount = 0, maxFollow = 1024; - auto ret = canonPathInner( + auto ret = canonPathInner>( path, [&followCount, &temp, maxFollow, resolveSymlinks] (std::string & result, std::string_view & remaining) { @@ -122,7 +122,7 @@ Path canonPath(PathView path, bool resolveSymlinks) Path dirOf(const PathView path) { - Path::size_type pos = NativePathTrait::rfindPathSep(path); + Path::size_type pos = OsPathTrait::rfindPathSep(path); if (pos == path.npos) return "."; return fs::path{path}.parent_path().string(); @@ -135,10 +135,10 @@ std::string_view baseNameOf(std::string_view path) return ""; auto last = path.size() - 1; - while (last > 0 && NativePathTrait::isPathSep(path[last])) + while (last > 0 && OsPathTrait::isPathSep(path[last])) last -= 1; - auto pos = NativePathTrait::rfindPathSep(path, last); + auto pos = OsPathTrait::rfindPathSep(path, last); if (pos == path.npos) pos = 0; else @@ -569,7 +569,7 @@ void replaceSymlink(const Path & target, const Path & link) } void setWriteTime( - const std::filesystem::path & path, + const fs::path & path, time_t accessedTime, time_t modificationTime, std::optional optIsSymlink) @@ -685,4 +685,17 @@ void moveFile(const Path & oldName, const Path & newName) ////////////////////////////////////////////////////////////////////// +bool isExecutableFileAmbient(const fs::path & exe) { + // Check file type, because directory being executable means + // something completely different. + return std::filesystem::is_regular_file(exe) + && access(exe.string().c_str(), +#ifdef WIN32 + 0 // TODO do better +#else + X_OK +#endif + ) == 0; +} + } diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh index 4b215162d..5350ceb7e 100644 --- a/src/libutil/file-system.hh +++ b/src/libutil/file-system.hh @@ -263,6 +263,12 @@ std::pair createTempFile(const Path & prefix = "nix"); */ Path defaultTempDir(); +/** + * Interpret `exe` as a location in the ambient file system and return + * whether it exists AND is executable. + */ +bool isExecutableFileAmbient(const std::filesystem::path & exe); + /** * Used in various places. */ diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 8552c4c9d..8ff7ee51f 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -129,6 +129,7 @@ sources = files( 'english.cc', 'environment-variables.cc', 'error.cc', + 'executable-path.cc', 'exit.cc', 'experimental-features.cc', 'file-content-address.cc', @@ -183,6 +184,7 @@ headers = [config_h] + files( 'english.hh', 'environment-variables.hh', 'error.hh', + 'executable-path.hh', 'exit.hh', 'experimental-features.hh', 'file-content-address.hh', @@ -202,6 +204,7 @@ headers = [config_h] + files( 'lru-cache.hh', 'memory-source-accessor.hh', 'muxable-pipe.hh', + 'os-string.hh', 'pool.hh', 'position.hh', 'posix-source-accessor.hh', diff --git a/src/libutil/os-string.hh b/src/libutil/os-string.hh new file mode 100644 index 000000000..0d75173e5 --- /dev/null +++ b/src/libutil/os-string.hh @@ -0,0 +1,43 @@ +#pragma once +///@file + +#include +#include +#include + +namespace nix { + +/** + * Named because it is similar to the Rust type, except it is in the + * native encoding not WTF-8. + * + * Same as `std::filesystem::path::string_type`, but manually defined to + * avoid including a much more complex header. + */ +using OsString = std::basic_string< +#if defined(_WIN32) && !defined(__CYGWIN__) + wchar_t +#else + char +#endif + >; + +/** + * `std::string_view` counterpart for `OsString`. + */ +using OsStringView = std::basic_string_view; + +std::string os_string_to_string(OsStringView path); + +OsString string_to_os_string(std::string_view s); + +/** + * Create string literals with the native character width of paths + */ +#ifndef _WIN32 +# define OS_STR(s) s +#else +# define OS_STR(s) L##s +#endif + +} diff --git a/src/libutil/strings-inline.hh b/src/libutil/strings-inline.hh index d254d486d..25b8e0ff6 100644 --- a/src/libutil/strings-inline.hh +++ b/src/libutil/strings-inline.hh @@ -4,8 +4,8 @@ namespace nix { -template -C tokenizeString(std::string_view s, std::string_view separators) +template +C basicTokenizeString(std::basic_string_view s, std::basic_string_view separators) { C result; auto pos = s.find_first_not_of(separators, 0); @@ -13,14 +13,42 @@ C tokenizeString(std::string_view s, std::string_view separators) auto end = s.find_first_of(separators, pos + 1); if (end == s.npos) end = s.size(); - result.insert(result.end(), std::string(s, pos, end - pos)); + result.insert(result.end(), std::basic_string(s, pos, end - pos)); pos = s.find_first_not_of(separators, end); } return result; } template -std::string concatStringsSep(const std::string_view sep, const C & ss) +C tokenizeString(std::string_view s, std::string_view separators) +{ + return basicTokenizeString(s, separators); +} + +template +C basicSplitString(std::basic_string_view s, std::basic_string_view separators) +{ + C result; + size_t pos = 0; + while (pos <= s.size()) { + auto end = s.find_first_of(separators, pos); + if (end == s.npos) + end = s.size(); + result.insert(result.end(), std::basic_string(s, pos, end - pos)); + pos = end + 1; + } + + return result; +} + +template +C splitString(std::string_view s, std::string_view separators) +{ + return basicSplitString(s, separators); +} + +template +std::basic_string basicConcatStringsSep(const std::basic_string_view sep, const C & ss) { size_t size = 0; bool tail = false; @@ -28,10 +56,10 @@ std::string concatStringsSep(const std::string_view sep, const C & ss) for (const auto & s : ss) { if (tail) size += sep.size(); - size += std::string_view(s).size(); + size += std::basic_string_view{s}.size(); tail = true; } - std::string s; + std::basic_string s; s.reserve(size); tail = false; for (auto & i : ss) { @@ -43,6 +71,12 @@ std::string concatStringsSep(const std::string_view sep, const C & ss) return s; } +template +std::string concatStringsSep(const std::string_view sep, const C & ss) +{ + return basicConcatStringsSep(sep, ss); +} + template std::string dropEmptyInitThenConcatStringsSep(const std::string_view sep, const C & ss) { diff --git a/src/libutil/strings.cc b/src/libutil/strings.cc index 2bb7f8c0a..60297228e 100644 --- a/src/libutil/strings.cc +++ b/src/libutil/strings.cc @@ -1,6 +1,8 @@ +#include #include #include "strings-inline.hh" +#include "os-string.hh" namespace nix { @@ -8,6 +10,13 @@ template std::list tokenizeString(std::string_view s, std::string_v template std::set tokenizeString(std::string_view s, std::string_view separators); template std::vector tokenizeString(std::string_view s, std::string_view separators); +template std::list splitString(std::string_view s, std::string_view separators); +template std::set splitString(std::string_view s, std::string_view separators); +template std::vector splitString(std::string_view s, std::string_view separators); + +template std::list basicSplitString( + std::basic_string_view s, std::basic_string_view separators); + template std::string concatStringsSep(std::string_view, const std::list &); template std::string concatStringsSep(std::string_view, const std::set &); template std::string concatStringsSep(std::string_view, const std::vector &); diff --git a/src/libutil/strings.hh b/src/libutil/strings.hh index 6e991e490..88b48d770 100644 --- a/src/libutil/strings.hh +++ b/src/libutil/strings.hh @@ -13,6 +13,12 @@ namespace nix { * * See also `basicSplitString()`, which preserves empty strings between separators, as well as at the start and end. */ +template +C basicTokenizeString(std::basic_string_view s, std::basic_string_view separators); + +/** + * Like `basicTokenizeString` but specialized to the default `char` + */ template C tokenizeString(std::string_view s, std::string_view separators = " \t\n\r"); @@ -20,6 +26,20 @@ extern template std::list tokenizeString(std::string_view s, std::s extern template std::set tokenizeString(std::string_view s, std::string_view separators); extern template std::vector tokenizeString(std::string_view s, std::string_view separators); +/** + * Split a string, preserving empty strings between separators, as well as at the start and end. + * + * Returns a non-empty collection of strings. + */ +template +C basicSplitString(std::basic_string_view s, std::basic_string_view separators); +template +C splitString(std::string_view s, std::string_view separators); + +extern template std::list splitString(std::string_view s, std::string_view separators); +extern template std::set splitString(std::string_view s, std::string_view separators); +extern template std::vector splitString(std::string_view s, std::string_view separators); + /** * Concatenate the given strings with a separator between the elements. */ diff --git a/src/libutil/unix/environment-variables.cc b/src/libutil/unix/environment-variables.cc index 9c6fd3b18..cd7c8f5e5 100644 --- a/src/libutil/unix/environment-variables.cc +++ b/src/libutil/unix/environment-variables.cc @@ -9,4 +9,14 @@ int setEnv(const char * name, const char * value) return ::setenv(name, value, 1); } +std::optional getEnvOs(const std::string & key) +{ + return getEnv(key); +} + +int setEnvOs(const OsString & name, const OsString & value) +{ + return setEnv(name.c_str(), value.c_str()); +} + } diff --git a/src/libutil/unix/file-path.cc b/src/libutil/unix/file-path.cc index 294048a2f..cccee86a1 100644 --- a/src/libutil/unix/file-path.cc +++ b/src/libutil/unix/file-path.cc @@ -8,16 +8,6 @@ namespace nix { -std::string os_string_to_string(PathViewNG::string_view path) -{ - return std::string { path }; -} - -std::filesystem::path::string_type string_to_os_string(std::string_view s) -{ - return std::string { s }; -} - std::optional maybePath(PathView path) { return { path }; diff --git a/src/libutil/unix/meson.build b/src/libutil/unix/meson.build index 38e5cd3aa..1c5bf27fb 100644 --- a/src/libutil/unix/meson.build +++ b/src/libutil/unix/meson.build @@ -4,6 +4,7 @@ sources += files( 'file-path.cc', 'file-system.cc', 'muxable-pipe.cc', + 'os-string.cc', 'processes.cc', 'signals.cc', 'users.cc', diff --git a/src/libutil/unix/os-string.cc b/src/libutil/unix/os-string.cc new file mode 100644 index 000000000..8378afde2 --- /dev/null +++ b/src/libutil/unix/os-string.cc @@ -0,0 +1,21 @@ +#include +#include +#include +#include + +#include "file-path.hh" +#include "util.hh" + +namespace nix { + +std::string os_string_to_string(PathViewNG::string_view path) +{ + return std::string{path}; +} + +std::filesystem::path::string_type string_to_os_string(std::string_view s) +{ + return std::string{s}; +} + +} diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 460802be3..db3ed1ddf 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1,5 +1,6 @@ #include "util.hh" #include "fmt.hh" +#include "file-path.hh" #include #include diff --git a/src/libutil/windows/environment-variables.cc b/src/libutil/windows/environment-variables.cc index 25ab9d63a..525d08c64 100644 --- a/src/libutil/windows/environment-variables.cc +++ b/src/libutil/windows/environment-variables.cc @@ -4,7 +4,30 @@ namespace nix { -int unsetenv(const char *name) +std::optional getEnvOs(const OsString & key) +{ + // Determine the required buffer size for the environment variable value + DWORD bufferSize = GetEnvironmentVariableW(key.c_str(), nullptr, 0); + if (bufferSize == 0) { + return std::nullopt; + } + + // Allocate a buffer to hold the environment variable value + std::wstring value{L'\0', bufferSize}; + + // Retrieve the environment variable value + DWORD resultSize = GetEnvironmentVariableW(key.c_str(), &value[0], bufferSize); + if (resultSize == 0) { + return std::nullopt; + } + + // Resize the string to remove the extra null characters + value.resize(resultSize); + + return value; +} + +int unsetenv(const char * name) { return -SetEnvironmentVariableA(name, nullptr); } @@ -14,4 +37,9 @@ int setEnv(const char * name, const char * value) return -SetEnvironmentVariableA(name, value); } +int setEnvOs(const OsString & name, const OsString & value) +{ + return -SetEnvironmentVariableW(name.c_str(), value.c_str()); +} + } diff --git a/src/libutil/windows/file-path.cc b/src/libutil/windows/file-path.cc index 3114ac4df..7405c426b 100644 --- a/src/libutil/windows/file-path.cc +++ b/src/libutil/windows/file-path.cc @@ -9,18 +9,6 @@ namespace nix { -std::string os_string_to_string(PathViewNG::string_view path) -{ - std::wstring_convert> converter; - return converter.to_bytes(std::filesystem::path::string_type { path }); -} - -std::filesystem::path::string_type string_to_os_string(std::string_view s) -{ - std::wstring_convert> converter; - return converter.from_bytes(std::string { s }); -} - std::optional maybePath(PathView path) { if (path.length() >= 3 && (('A' <= path[0] && path[0] <= 'Z') || ('a' <= path[0] && path[0] <= 'z')) && path[1] == ':' && WindowsPathTrait::isPathSep(path[2])) { diff --git a/src/libutil/windows/meson.build b/src/libutil/windows/meson.build index 00320877f..1c645fe05 100644 --- a/src/libutil/windows/meson.build +++ b/src/libutil/windows/meson.build @@ -4,6 +4,7 @@ sources += files( 'file-path.cc', 'file-system.cc', 'muxable-pipe.cc', + 'os-string.cc', 'processes.cc', 'users.cc', 'windows-async-pipe.cc', diff --git a/src/libutil/windows/os-string.cc b/src/libutil/windows/os-string.cc new file mode 100644 index 000000000..7507f9030 --- /dev/null +++ b/src/libutil/windows/os-string.cc @@ -0,0 +1,24 @@ +#include +#include +#include +#include + +#include "file-path.hh" +#include "file-path-impl.hh" +#include "util.hh" + +namespace nix { + +std::string os_string_to_string(PathViewNG::string_view path) +{ + std::wstring_convert> converter; + return converter.to_bytes(std::filesystem::path::string_type{path}); +} + +std::filesystem::path::string_type string_to_os_string(std::string_view s) +{ + std::wstring_convert> converter; + return converter.from_bytes(std::string{s}); +} + +} diff --git a/src/nix/config-check.cc b/src/nix/config-check.cc index 09d140733..1a6574de2 100644 --- a/src/nix/config-check.cc +++ b/src/nix/config-check.cc @@ -8,6 +8,7 @@ #include "store-api.hh" #include "local-fs-store.hh" #include "worker-protocol.hh" +#include "executable-path.hh" using namespace nix; @@ -39,6 +40,8 @@ void checkInfo(const std::string & msg) { } +namespace fs = std::filesystem; + struct CmdConfigCheck : StoreCommand { bool success = true; @@ -75,11 +78,13 @@ struct CmdConfigCheck : StoreCommand bool checkNixInPath() { - PathSet dirs; + std::set dirs; - for (auto & dir : tokenizeString(getEnv("PATH").value_or(""), ":")) - if (pathExists(dir + "/nix-env")) - dirs.insert(dirOf(canonPath(dir + "/nix-env", true))); + for (auto & dir : ExecutablePath::load().directories) { + auto candidate = dir / "nix-env"; + if (fs::exists(candidate)) + dirs.insert(fs::canonical(candidate).parent_path() ); + } if (dirs.size() != 1) { std::stringstream ss; @@ -94,18 +99,25 @@ struct CmdConfigCheck : StoreCommand bool checkProfileRoots(ref store) { - PathSet dirs; + std::set dirs; - for (auto & dir : tokenizeString(getEnv("PATH").value_or(""), ":")) { - Path profileDir = dirOf(dir); + for (auto & dir : ExecutablePath::load().directories) { + auto profileDir = dir.parent_path(); try { - Path userEnv = canonPath(profileDir, true); + auto userEnv = fs::weakly_canonical(profileDir); - if (store->isStorePath(userEnv) && hasSuffix(userEnv, "user-environment")) { - while (profileDir.find("/profiles/") == std::string::npos && std::filesystem::is_symlink(profileDir)) - profileDir = absPath(readLink(profileDir), dirOf(profileDir)); + auto noContainsProfiles = [&]{ + for (auto && part : profileDir) + if (part == "profiles") return false; + return true; + }; - if (profileDir.find("/profiles/") == std::string::npos) + if (store->isStorePath(userEnv.string()) && hasSuffix(userEnv.string(), "user-environment")) { + while (noContainsProfiles() && std::filesystem::is_symlink(profileDir)) + profileDir = fs::weakly_canonical( + profileDir.parent_path() / fs::read_symlink(profileDir)); + + if (noContainsProfiles()) dirs.insert(dir); } } catch (SystemError &) { diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 92ec3b78a..effc86a0a 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -415,7 +415,7 @@ struct Common : InstallableCommand, MixProfile if (buildEnvironment.providesStructuredAttrs()) { fixupStructuredAttrs( - PATHNG_LITERAL("sh"), + OS_STR("sh"), "NIX_ATTRS_SH_FILE", buildEnvironment.getAttrsSH(), rewrites, @@ -423,7 +423,7 @@ struct Common : InstallableCommand, MixProfile tmpDir ); fixupStructuredAttrs( - PATHNG_LITERAL("json"), + OS_STR("json"), "NIX_ATTRS_JSON_FILE", buildEnvironment.getAttrsJSON(), rewrites, @@ -447,7 +447,7 @@ struct Common : InstallableCommand, MixProfile const BuildEnvironment & buildEnvironment, const std::filesystem::path & tmpDir) { - auto targetFilePath = tmpDir / PATHNG_LITERAL(".attrs."); + auto targetFilePath = tmpDir / OS_STR(".attrs."); targetFilePath += ext; writeFile(targetFilePath.string(), content); diff --git a/src/nix/env.cc b/src/nix/env.cc index 9db03ca37..832320320 100644 --- a/src/nix/env.cc +++ b/src/nix/env.cc @@ -5,6 +5,7 @@ #include "eval.hh" #include "run.hh" #include "strings.hh" +#include "executable-path.hh" using namespace nix; @@ -95,10 +96,10 @@ struct CmdShell : InstallablesCommand, MixEnvironment } // TODO: split losslessly; empty means . - auto unixPath = tokenizeString(getEnv("PATH").value_or(""), ":"); - unixPath.insert(unixPath.begin(), pathAdditions.begin(), pathAdditions.end()); - auto unixPathString = concatStringsSep(":", unixPath); - setEnv("PATH", unixPathString.c_str()); + auto unixPath = ExecutablePath::load(); + unixPath.directories.insert(unixPath.directories.begin(), pathAdditions.begin(), pathAdditions.end()); + auto unixPathString = unixPath.render(); + setEnvOs(OS_STR("PATH"), unixPathString.c_str()); Strings args; for (auto & arg : command) diff --git a/src/nix/search.cc b/src/nix/search.cc index 7f8504d3f..c8d0b9e96 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -10,6 +10,7 @@ #include "eval-cache.hh" #include "attr-path.hh" #include "hilite.hh" +#include "strings-inline.hh" #include #include diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 7b3357700..9ca3f6087 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -8,6 +8,7 @@ #include "attr-path.hh" #include "names.hh" #include "progress-bar.hh" +#include "executable-path.hh" using namespace nix; @@ -102,23 +103,17 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand /* Return the profile in which Nix is installed. */ Path getProfileDir(ref store) { - Path where; - - for (auto & dir : tokenizeString(getEnv("PATH").value_or(""), ":")) - if (pathExists(dir + "/nix-env")) { - where = dir; - break; - } - - if (where == "") + auto whereOpt = ExecutablePath::load().find(OS_STR("nix-env")); + if (!whereOpt) throw Error("couldn't figure out how Nix is installed, so I can't upgrade it"); + auto & where = *whereOpt; printInfo("found Nix in '%s'", where); - if (hasPrefix(where, "/run/current-system")) + if (hasPrefix(where.string(), "/run/current-system")) throw Error("Nix on NixOS must be upgraded via 'nixos-rebuild'"); - Path profileDir = dirOf(where); + Path profileDir = where.parent_path().string(); // Resolve profile to /nix/var/nix/profiles/ link. while (canonPath(profileDir).find("/profiles/") == std::string::npos && std::filesystem::is_symlink(profileDir)) @@ -128,7 +123,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand Path userEnv = canonPath(profileDir, true); - if (baseNameOf(where) != "bin" || + if (where.filename() != "bin" || !hasSuffix(userEnv, "user-environment")) throw Error("directory '%s' does not appear to be part of a Nix profile", where); diff --git a/tests/unit/libutil/executable-path.cc b/tests/unit/libutil/executable-path.cc new file mode 100644 index 000000000..8d182357d --- /dev/null +++ b/tests/unit/libutil/executable-path.cc @@ -0,0 +1,64 @@ +#include + +#include "executable-path.hh" + +namespace nix { + +#ifdef WIN32 +# define PATH_VAR_SEP L";" +#else +# define PATH_VAR_SEP ":" +#endif + +#define PATH_ENV_ROUND_TRIP(NAME, STRING_LIT, CXX_LIT) \ + TEST(ExecutablePath, NAME) \ + { \ + OsString s = STRING_LIT; \ + auto v = ExecutablePath::parse(s); \ + EXPECT_EQ(v, (ExecutablePath CXX_LIT)); \ + auto s2 = v.render(); \ + EXPECT_EQ(s2, s); \ + } + +PATH_ENV_ROUND_TRIP(emptyRoundTrip, OS_STR(""), ({})) + +PATH_ENV_ROUND_TRIP( + oneElemRoundTrip, + OS_STR("/foo"), + ({ + OS_STR("/foo"), + })) + +PATH_ENV_ROUND_TRIP( + twoElemsRoundTrip, + OS_STR("/foo" PATH_VAR_SEP "/bar"), + ({ + OS_STR("/foo"), + OS_STR("/bar"), + })) + +PATH_ENV_ROUND_TRIP( + threeElemsRoundTrip, + OS_STR("/foo" PATH_VAR_SEP "." PATH_VAR_SEP "/bar"), + ({ + OS_STR("/foo"), + OS_STR("."), + OS_STR("/bar"), + })) + +TEST(ExecutablePath, elementyElemNormalize) +{ + auto v = ExecutablePath::parse(PATH_VAR_SEP PATH_VAR_SEP PATH_VAR_SEP); + EXPECT_EQ( + v, + (ExecutablePath{{ + OS_STR("."), + OS_STR("."), + OS_STR("."), + OS_STR("."), + }})); + auto s2 = v.render(); + EXPECT_EQ(s2, OS_STR("." PATH_VAR_SEP "." PATH_VAR_SEP "." PATH_VAR_SEP ".")); +} + +} diff --git a/tests/unit/libutil/meson.build b/tests/unit/libutil/meson.build index f4f2ae7f9..83cec13ec 100644 --- a/tests/unit/libutil/meson.build +++ b/tests/unit/libutil/meson.build @@ -52,6 +52,7 @@ sources = files( 'closure.cc', 'compression.cc', 'config.cc', + 'executable-path.cc', 'file-content-address.cc', 'git.cc', 'hash.cc', diff --git a/tests/unit/libutil/strings.cc b/tests/unit/libutil/strings.cc index 0bd2fe0a5..8ceb16767 100644 --- a/tests/unit/libutil/strings.cc +++ b/tests/unit/libutil/strings.cc @@ -1,4 +1,5 @@ #include +#include #include "strings.hh" @@ -231,4 +232,117 @@ TEST(tokenizeString, tokenizeSepEmpty) ASSERT_EQ(tokenizeString(s, ","), expected); } +/* ---------------------------------------------------------------------------- + * splitString + * --------------------------------------------------------------------------*/ + +TEST(splitString, empty) +{ + Strings expected = {""}; + + ASSERT_EQ(splitString("", " \t\n\r"), expected); +} + +TEST(splitString, oneSep) +{ + Strings expected = {"", ""}; + + ASSERT_EQ(splitString(" ", " \t\n\r"), expected); +} + +TEST(splitString, twoSep) +{ + Strings expected = {"", "", ""}; + + ASSERT_EQ(splitString(" \n", " \t\n\r"), expected); +} + +TEST(splitString, tokenizeSpacesWithSpaces) +{ + auto s = "foo bar baz"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(splitString(s, " \t\n\r"), expected); +} + +TEST(splitString, tokenizeTabsWithDefaults) +{ + auto s = "foo\tbar\tbaz"; + // Using it like this is weird, but shows the difference with tokenizeString, which also has this test + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(splitString(s, " \t\n\r"), expected); +} + +TEST(splitString, tokenizeTabsSpacesWithDefaults) +{ + auto s = "foo\t bar\t baz"; + // Using it like this is weird, but shows the difference with tokenizeString, which also has this test + Strings expected = {"foo", "", "bar", "", "baz"}; + + ASSERT_EQ(splitString(s, " \t\n\r"), expected); +} + +TEST(splitString, tokenizeTabsSpacesNewlineWithDefaults) +{ + auto s = "foo\t\n bar\t\n baz"; + // Using it like this is weird, but shows the difference with tokenizeString, which also has this test + Strings expected = {"foo", "", "", "bar", "", "", "baz"}; + + ASSERT_EQ(splitString(s, " \t\n\r"), expected); +} + +TEST(splitString, tokenizeTabsSpacesNewlineRetWithDefaults) +{ + auto s = "foo\t\n\r bar\t\n\r baz"; + // Using it like this is weird, but shows the difference with tokenizeString, which also has this test + Strings expected = {"foo", "", "", "", "bar", "", "", "", "baz"}; + + ASSERT_EQ(splitString(s, " \t\n\r"), expected); + + auto s2 = "foo \t\n\r bar \t\n\r baz"; + Strings expected2 = {"foo", "", "", "", "", "bar", "", "", "", "", "baz"}; + + ASSERT_EQ(splitString(s2, " \t\n\r"), expected2); +} + +TEST(splitString, tokenizeWithCustomSep) +{ + auto s = "foo\n,bar\n,baz\n"; + Strings expected = {"foo\n", "bar\n", "baz\n"}; + + ASSERT_EQ(splitString(s, ","), expected); +} + +TEST(splitString, tokenizeSepAtStart) +{ + auto s = ",foo,bar,baz"; + Strings expected = {"", "foo", "bar", "baz"}; + + ASSERT_EQ(splitString(s, ","), expected); +} + +TEST(splitString, tokenizeSepAtEnd) +{ + auto s = "foo,bar,baz,"; + Strings expected = {"foo", "bar", "baz", ""}; + + ASSERT_EQ(splitString(s, ","), expected); +} + +TEST(splitString, tokenizeSepEmpty) +{ + auto s = "foo,,baz"; + Strings expected = {"foo", "", "baz"}; + + ASSERT_EQ(splitString(s, ","), expected); +} + +// concatStringsSep sep . splitString sep = id if sep is 1 char +RC_GTEST_PROP(splitString, recoveredByConcatStringsSep, (const std::string & s)) +{ + RC_ASSERT(concatStringsSep("/", splitString(s, "/")) == s); + RC_ASSERT(concatStringsSep("a", splitString(s, "a")) == s); +} + } // namespace nix From 0646b6cd610d901438fb66eadf09708d302e1c66 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 7 Aug 2024 22:29:56 -0500 Subject: [PATCH 124/142] Update comments / documentation. Co-authored-by: Robert Hensing --- src/libutil/file-system.cc | 1 + src/libutil/file-system.hh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 156dddf98..536ae29ab 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -688,6 +688,7 @@ void moveFile(const Path & oldName, const Path & newName) bool isExecutableFileAmbient(const fs::path & exe) { // Check file type, because directory being executable means // something completely different. + // `is_regular_file` follows symlinks before checking. return std::filesystem::is_regular_file(exe) && access(exe.string().c_str(), #ifdef WIN32 diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh index 5350ceb7e..1ae5fa136 100644 --- a/src/libutil/file-system.hh +++ b/src/libutil/file-system.hh @@ -265,7 +265,7 @@ Path defaultTempDir(); /** * Interpret `exe` as a location in the ambient file system and return - * whether it exists AND is executable. + * whether it resolves to a file that is executable. */ bool isExecutableFileAmbient(const std::filesystem::path & exe); From 3cc2e2a0ac7df557f5cd958a0736ce4eedf5ae07 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 11 Aug 2024 03:31:44 +0200 Subject: [PATCH 125/142] Edit docs --- doc/manual/rl-next/ban-integer-overflow.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/manual/rl-next/ban-integer-overflow.md b/doc/manual/rl-next/ban-integer-overflow.md index 49ecfe4c6..0e553af76 100644 --- a/doc/manual/rl-next/ban-integer-overflow.md +++ b/doc/manual/rl-next/ban-integer-overflow.md @@ -4,10 +4,10 @@ issues: [10968] prs: [11188] --- -Previously, integer overflow in the Nix language invoked C++ level signed overflow, which was undefined behaviour, but *probably* manifested as wrapping around on overflow. +Previously, integer overflow in the Nix language invoked C++ level signed overflow, which was undefined behaviour, but *usually* manifested as wrapping around on overflow. Since prior to the public release of Lix, Lix had C++ signed overflow defined to crash the process and nobody noticed this having accidentally removed overflow from the Nix language for three months until it was caught by fiddling around. -Given the significant body of actual Nix code that has been evaluated by Lix in that time, it does not appear that nixpkgs or much of importance depends on integer overflow, so it is safe to turn into an error. +Given the significant body of actual Nix code that has been evaluated by Lix in that time, it does not appear that nixpkgs or much of importance depends on integer overflow, so it appears safe to turn into an error. Some other overflows were fixed: - `builtins.fromJSON` of values greater than the maximum representable value in a signed 64-bit integer will generate an error. @@ -16,6 +16,6 @@ Some other overflows were fixed: Integer overflow now looks like the following: ``` -» nix eval --expr '9223372036854775807 + 1' +$ nix eval --expr '9223372036854775807 + 1' error: integer overflow in adding 9223372036854775807 + 1 ``` From f7c86d1a2f4460f58e8a91d004e438e69a3e133c Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 11 Aug 2024 12:47:35 +0200 Subject: [PATCH 126/142] CONTRIBUTING.md: Add attribution and context rules We've recently had an incident where these rules were not followed, so let's add guidelines to increase the chances of contributors getting this right. Relevant discussion: https://discourse.nixos.org/t/code-attribution-policy/50445/2 --- CONTRIBUTING.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 12423366a..1c9ee9b35 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -52,6 +52,20 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). Link related issues to inform interested parties and future contributors about your change. If your pull request closes one or multiple issues, mention that in the description using `Closes: #`, as it will then happen automatically when your change is merged. + * Credit original authors when you're reusing or building on their work. + * Link to relevant changes in other projects, so that others can understand the full context of the change in the future when you or someone else will change or troubleshoot the code. + This is especially important when your change is based on work done in other repositories. + + Example: + ``` + This is based on the work of @user in . + This solution took inspiration from . + + Co-authored-by: User Name + ``` + + Use the `git cherry-pick -x` flag, and amend the commits to link to forks when applicable. + * Make sure to have [a clean history of commits on your branch by using rebase](https://www.digitalocean.com/community/tutorials/how-to-rebase-and-update-a-pull-request). * [Mark the pull request as draft](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/changing-the-stage-of-a-pull-request) if you're not done with the changes. From b64d6aa7b0fcc801d8ee496c41658d9883af37aa Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 11 Aug 2024 18:07:19 +0200 Subject: [PATCH 127/142] CONTRIBUTING.md: Clarify use of cherry-pick on forks --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1c9ee9b35..da56653b8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,7 +64,7 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). Co-authored-by: User Name ``` - Use the `git cherry-pick -x` flag, and amend the commits to link to forks when applicable. + When cherry-picking from a different repository, use the `-x` flag, and then amend the commits to turn the hashes into URLs. * Make sure to have [a clean history of commits on your branch by using rebase](https://www.digitalocean.com/community/tutorials/how-to-rebase-and-update-a-pull-request). * [Mark the pull request as draft](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/changing-the-stage-of-a-pull-request) if you're not done with the changes. From 58b03ef1cd73385699e3ea5c46d0dc0d36c3bbcd Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 24 Jul 2024 23:14:36 -0400 Subject: [PATCH 128/142] Move `NIX_BIN_DIR` and all logic using it to the Nix executable itself This is because with the split packages of the Meson build, we simply have no idea what directory the binaries will be installed in when we build the library. In the process of doing so, consolidate and make more sophisticated the logic to cope with a few corner cases (e.g. `NIX_BIN_DIR` exists, but no binaries are inside it). Co-authored-by: Robert Hensing --- doc/manual/rl-next/build-hook-default.md | 22 ++++++++++++++ src/libcmd/repl.cc | 38 +++++++++++------------- src/libcmd/repl.hh | 14 +++++++-- src/libstore/globals.cc | 29 ------------------ src/libstore/globals.hh | 7 +---- src/libstore/local.mk | 1 - src/libstore/meson.build | 2 -- src/libstore/unix/build/hook-instance.cc | 14 +++++++-- src/libutil/executable-path.cc | 18 ++++++++++- src/libutil/executable-path.hh | 15 +++++++++- src/nix-channel/nix-channel.cc | 11 +++---- src/nix/local.mk | 2 ++ src/nix/main.cc | 12 ++++++++ src/nix/meson.build | 19 +++++++++++- src/nix/repl.cc | 23 +++++++++++++- src/nix/self-exe.cc | 38 ++++++++++++++++++++++++ src/nix/self-exe.hh | 31 +++++++++++++++++++ src/nix/upgrade-nix.cc | 5 ++-- src/perl/lib/Nix/Config.pm.in | 1 - src/perl/lib/Nix/Store.pm | 2 +- src/perl/lib/Nix/Store.xs | 5 ---- 21 files changed, 227 insertions(+), 82 deletions(-) create mode 100644 doc/manual/rl-next/build-hook-default.md create mode 100644 src/nix/self-exe.cc create mode 100644 src/nix/self-exe.hh diff --git a/doc/manual/rl-next/build-hook-default.md b/doc/manual/rl-next/build-hook-default.md new file mode 100644 index 000000000..197290536 --- /dev/null +++ b/doc/manual/rl-next/build-hook-default.md @@ -0,0 +1,22 @@ +--- +synopsis: |- + The `build-hook` setting's default is less useful when using `libnixstore` as a library +prs: +- 11178 +--- + +*This is an obscure issue that only affects usage of the `libnixstore` library outside of the Nix executable.* + +As part the ongoing [rewrite of the build system](https://github.com/NixOS/nix/issues/2503) to use [Meson](https://mesonbuild.com/), we are also switching to packaging individual Nix components separately (and building them in separate derivations). +This means that when building `libnixstore` we do not know where the Nix binaries will be installed --- `libnixstore` doesn't know about downstream consumers like the Nix binaries at all. + +*This is also unrelated to the _`post`_-`build-hook`*, which is often used for pushing to a cache.* + +This has a small adverse affect on remote building --- the `build-remote` executable that is specified from the [`build-hook`](@docroot@/command-ref/conf-file.md#conf-build-hook) setting will not be gotten from the (presumed) installation location, but instead looked up on the `PATH`. +This means that other applications linking `libnixstore` that wish to use remote building must arrange for the `nix` command to be on the PATH (or manually overriding `build-hook`) in order for that to work. + +Long term we don't envision this being a downside, because we plan to [get rid of `build-remote` and the build hook setting entirely](https://github.com/NixOS/nix/issues/1221). +There is simply no need to add a second layer of remote-procedure-calling when we want to connect to a remote builder. +The build hook protocol did in principle support custom ways of remote building, but that can also be accomplished with a custom service for the ssh or daemon/ssh-ng protocols, or with a custom [store type](@docroot@/store/types/) i.e. `Store` subclass. + +The Perl bindings no longer expose `getBinDir` either, since they libraries those bindings wrap no longer know the location of installed binaries as described above. diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index efc04b029..e7c43367c 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -7,7 +7,6 @@ #include "ansicolor.hh" #include "shared.hh" -#include "config-global.hh" #include "eval.hh" #include "eval-settings.hh" #include "attr-path.hh" @@ -77,10 +76,14 @@ struct NixRepl int displ; StringSet varNames; + RunNix * runNixPtr; + + void runNix(Path program, const Strings & args, const std::optional & input = {}); + std::unique_ptr interacter; NixRepl(const LookupPath & lookupPath, nix::ref store,ref state, - std::function getValues); + std::function getValues, RunNix * runNix); virtual ~NixRepl() = default; ReplExitStatus mainLoop() override; @@ -125,32 +128,16 @@ std::string removeWhitespace(std::string s) NixRepl::NixRepl(const LookupPath & lookupPath, nix::ref store, ref state, - std::function getValues) + std::function getValues, RunNix * runNix = nullptr) : AbstractNixRepl(state) , debugTraceIndex(0) , getValues(getValues) , staticEnv(new StaticEnv(nullptr, state->staticBaseEnv.get())) + , runNixPtr{runNix} , interacter(make_unique(getDataDir() + "/nix/repl-history")) { } -void runNix(Path program, const Strings & args, - const std::optional & input = {}) -{ - auto subprocessEnv = getEnv(); - subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue(); - //isInteractive avoid grabling interactive commands - runProgram2(RunOptions { - .program = settings.nixBinDir+ "/" + program, - .args = args, - .environment = subprocessEnv, - .input = input, - .isInteractive = true, - }); - - return; -} - static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positions, const DebugTrace & dt) { if (dt.isError) @@ -833,9 +820,18 @@ void NixRepl::evalString(std::string s, Value & v) } +void NixRepl::runNix(Path program, const Strings & args, const std::optional & input) +{ + if (runNixPtr) + (*runNixPtr)(program, args, input); + else + throw Error("Cannot run '%s', no method of calling the Nix CLI provided", program); +} + + std::unique_ptr AbstractNixRepl::create( const LookupPath & lookupPath, nix::ref store, ref state, - std::function getValues) + std::function getValues, RunNix * runNix) { return std::make_unique( lookupPath, diff --git a/src/libcmd/repl.hh b/src/libcmd/repl.hh index 3fd4b2c39..11d1820f5 100644 --- a/src/libcmd/repl.hh +++ b/src/libcmd/repl.hh @@ -19,9 +19,19 @@ struct AbstractNixRepl typedef std::vector> AnnotatedValues; + using RunNix = void(Path program, const Strings & args, const std::optional & input); + + /** + * @param runNix Function to run the nix CLI to support various + * `:` commands. Optional; if not provided, + * everything else will still work fine, but those commands won't. + */ static std::unique_ptr create( - const LookupPath & lookupPath, nix::ref store, ref state, - std::function getValues); + const LookupPath & lookupPath, + nix::ref store, + ref state, + std::function getValues, + RunNix * runNix = nullptr); static ReplExitStatus runSimple( ref evalState, diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 439a6f97c..52ab35b4c 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -64,7 +64,6 @@ Settings::Settings() , nixStateDir(canonPath(getEnvNonEmpty("NIX_STATE_DIR").value_or(NIX_STATE_DIR))) , nixConfDir(canonPath(getEnvNonEmpty("NIX_CONF_DIR").value_or(NIX_CONF_DIR))) , nixUserConfFiles(getUserConfigFiles()) - , nixBinDir(canonPath(getEnvNonEmpty("NIX_BIN_DIR").value_or(NIX_BIN_DIR))) , nixManDir(canonPath(NIX_MAN_DIR)) , nixDaemonSocketFile(canonPath(getEnvNonEmpty("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH))) { @@ -95,34 +94,6 @@ Settings::Settings() sandboxPaths = tokenizeString("/System/Library/Frameworks /System/Library/PrivateFrameworks /bin/sh /bin/bash /private/tmp /private/var/tmp /usr/lib"); allowedImpureHostPrefixes = tokenizeString("/System/Library /usr/lib /dev /bin/sh"); #endif - - /* Set the build hook location - - For builds we perform a self-invocation, so Nix has to be self-aware. - That is, it has to know where it is installed. We don't think it's sentient. - - Normally, nix is installed according to `nixBinDir`, which is set at compile time, - but can be overridden. This makes for a great default that works even if this - code is linked as a library into some other program whose main is not aware - that it might need to be a build remote hook. - - However, it may not have been installed at all. For example, if it's a static build, - there's a good chance that it has been moved out of its installation directory. - That makes `nixBinDir` useless. Instead, we'll query the OS for the path to the - current executable, using `getSelfExe()`. - - As a last resort, we resort to `PATH`. Hopefully we find a `nix` there that's compatible. - If you're porting Nix to a new platform, that might be good enough for a while, but - you'll want to improve `getSelfExe()` to work on your platform. - */ - std::string nixExePath = nixBinDir + "/nix"; - if (!pathExists(nixExePath)) { - nixExePath = getSelfExe().value_or("nix"); - } - buildHook = { - nixExePath, - "__build-remote", - }; } void loadConfFile(AbstractConfig & config) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 1ca9f02f5..65c7bf3bc 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -84,11 +84,6 @@ public: */ std::vector nixUserConfFiles; - /** - * The directory where the main programs are stored. - */ - Path nixBinDir; - /** * The directory where the man pages are stored. */ @@ -246,7 +241,7 @@ public: )", {"build-timeout"}}; - Setting buildHook{this, {}, "build-hook", + Setting buildHook{this, {"nix", "__build-remote"}, "build-hook", R"( The path to the helper program that executes remote builds. diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 5dc8f3370..88be6a366 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -71,7 +71,6 @@ libstore_CXXFLAGS += \ -DNIX_STATE_DIR=\"$(NIX_ROOT)$(localstatedir)/nix\" \ -DNIX_LOG_DIR=\"$(NIX_ROOT)$(localstatedir)/log/nix\" \ -DNIX_CONF_DIR=\"$(NIX_ROOT)$(sysconfdir)/nix\" \ - -DNIX_BIN_DIR=\"$(NIX_ROOT)$(bindir)\" \ -DNIX_MAN_DIR=\"$(NIX_ROOT)$(mandir)\" \ -DLSOF=\"$(NIX_ROOT)$(lsof)\" diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 50b15e15d..d2cc235fd 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -328,7 +328,6 @@ prefix = get_option('prefix') path_opts = [ # Meson built-ins. 'datadir', - 'bindir', 'mandir', 'libdir', 'includedir', @@ -373,7 +372,6 @@ cpp_str_defines = { 'NIX_STATE_DIR': state_dir / 'nix', 'NIX_LOG_DIR': log_dir, 'NIX_CONF_DIR': sysconfdir / 'nix', - 'NIX_BIN_DIR': bindir, 'NIX_MAN_DIR': mandir, } diff --git a/src/libstore/unix/build/hook-instance.cc b/src/libstore/unix/build/hook-instance.cc index d73d86ff2..4f8492fe9 100644 --- a/src/libstore/unix/build/hook-instance.cc +++ b/src/libstore/unix/build/hook-instance.cc @@ -4,6 +4,7 @@ #include "file-system.hh" #include "child.hh" #include "strings.hh" +#include "executable-path.hh" namespace nix { @@ -16,11 +17,18 @@ HookInstance::HookInstance() if (buildHookArgs.empty()) throw Error("'build-hook' setting is empty"); - auto buildHook = canonPath(buildHookArgs.front()); + std::filesystem::path buildHook = buildHookArgs.front(); buildHookArgs.pop_front(); + try { + buildHook = ExecutablePath::load().findPath(buildHook); + } catch (ExecutableLookupError & e) { + e.addTrace(nullptr, "while resolving the 'build-hook' setting'"); + throw; + } + Strings args; - args.push_back(std::string(baseNameOf(buildHook))); + args.push_back(buildHook.filename().string()); for (auto & arg : buildHookArgs) args.push_back(arg); @@ -59,7 +67,7 @@ HookInstance::HookInstance() if (dup2(builderOut.readSide.get(), 5) == -1) throw SysError("dupping builder's stdout/stderr"); - execv(buildHook.c_str(), stringsToCharPtrs(args).data()); + execv(buildHook.native().c_str(), stringsToCharPtrs(args).data()); throw SysError("executing '%s'", buildHook); }); diff --git a/src/libutil/executable-path.cc b/src/libutil/executable-path.cc index 1658e3667..8005a19be 100644 --- a/src/libutil/executable-path.cc +++ b/src/libutil/executable-path.cc @@ -60,7 +60,7 @@ OsString ExecutablePath::render() const } std::optional -ExecutablePath::find(const OsString & exe, std::function isExecutable) const +ExecutablePath::findName(const OsString & exe, std::function isExecutable) const { // "If the pathname being sought contains a , the search // through the path prefixes shall not be performed." @@ -76,4 +76,20 @@ ExecutablePath::find(const OsString & exe, std::function return std::nullopt; } +fs::path ExecutablePath::findPath(const fs::path & exe, std::function isExecutable) const +{ + // "If the pathname being sought contains a , the search + // through the path prefixes shall not be performed." + // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03 + if (exe.filename() == exe) { + auto resOpt = findName(exe, isExecutable); + if (resOpt) + return *resOpt; + else + throw ExecutableLookupError("Could not find executable '%s'", exe.native()); + } else { + return exe; + } +} + } // namespace nix diff --git a/src/libutil/executable-path.hh b/src/libutil/executable-path.hh index bcb8d28e8..f46d5e212 100644 --- a/src/libutil/executable-path.hh +++ b/src/libutil/executable-path.hh @@ -5,6 +5,8 @@ namespace nix { +MakeError(ExecutableLookupError, Error); + struct ExecutablePath { std::vector directories; @@ -54,10 +56,21 @@ struct ExecutablePath * * @return path to a resolved executable */ - std::optional find( + std::optional findName( const OsString & exe, std::function isExecutableFile = isExecutableFileAmbient) const; + /** + * Like the `findName` but also allows a file path as input. + * + * This implements the full POSIX spec: if the path is just a name, + * it searches like the above. Otherwise, it returns the path as is. + * If (in the name case) the search fails, an exception is thrown. + */ + std::filesystem::path findPath( + const std::filesystem::path & exe, + std::function isExecutable = isExecutableFileAmbient) const; + bool operator==(const ExecutablePath &) const = default; }; diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 9f7f557b5..878f3c828 100644 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -7,6 +7,7 @@ #include "eval-settings.hh" // for defexpr #include "users.hh" #include "tarball.hh" +#include "self-exe.hh" #include #include @@ -67,7 +68,7 @@ static void removeChannel(const std::string & name) channels.erase(name); writeChannels(); - runProgram(settings.nixBinDir + "/nix-env", true, { "--profile", profile, "--uninstall", name }); + runProgram(getNixBin("nix-env").string(), true, { "--profile", profile, "--uninstall", name }); } static Path nixDefExpr; @@ -118,7 +119,7 @@ static void update(const StringSet & channelNames) bool unpacked = false; if (std::regex_search(filename, std::regex("\\.tar\\.(gz|bz2|xz)$"))) { - runProgram(settings.nixBinDir + "/nix-build", false, { "--no-out-link", "--expr", "import " + unpackChannelPath + + runProgram(getNixBin("nix-build").string(), false, { "--no-out-link", "--expr", "import " + unpackChannelPath + "{ name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; }" }); unpacked = true; } @@ -143,7 +144,7 @@ static void update(const StringSet & channelNames) for (auto & expr : exprs) envArgs.push_back(std::move(expr)); envArgs.push_back("--quiet"); - runProgram(settings.nixBinDir + "/nix-env", false, envArgs); + runProgram(getNixBin("nix-env").string(), false, envArgs); // Make the channels appear in nix-env. struct stat st; @@ -244,7 +245,7 @@ static int main_nix_channel(int argc, char ** argv) case cListGenerations: if (!args.empty()) throw UsageError("'--list-generations' expects no arguments"); - std::cout << runProgram(settings.nixBinDir + "/nix-env", false, {"--profile", profile, "--list-generations"}) << std::flush; + std::cout << runProgram(getNixBin("nix-env").string(), false, {"--profile", profile, "--list-generations"}) << std::flush; break; case cRollback: if (args.size() > 1) @@ -256,7 +257,7 @@ static int main_nix_channel(int argc, char ** argv) } else { envArgs.push_back("--rollback"); } - runProgram(settings.nixBinDir + "/nix-env", false, envArgs); + runProgram(getNixBin("nix-env").string(), false, envArgs); break; } diff --git a/src/nix/local.mk b/src/nix/local.mk index 28b30b586..b57f6b3e2 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -26,6 +26,8 @@ endif nix_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) $(INCLUDE_libflake) $(INCLUDE_libmain) -I src/libcmd -I doc/manual $(INCLUDE_nix) +nix_CXXFLAGS += -DNIX_BIN_DIR=\"$(NIX_ROOT)$(bindir)\" + nix_LIBS = libexpr libmain libfetchers libflake libstore libutil libcmd nix_LDFLAGS = $(THREAD_LDFLAGS) $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) $(LOWDOWN_LIBS) diff --git a/src/nix/main.cc b/src/nix/main.cc index 9d7d617cc..34de79ac8 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -19,6 +19,7 @@ #include "network-proxy.hh" #include "eval-cache.hh" #include "flake/flake.hh" +#include "self-exe.hh" #include #include @@ -366,6 +367,17 @@ void mainWrapped(int argc, char * * argv) initGC(); flake::initLib(flakeSettings); + /* Set the build hook location + + For builds we perform a self-invocation, so Nix has to be + self-aware. That is, it has to know where it is installed. We + don't think it's sentient. + */ + settings.buildHook.setDefault(Strings { + getNixBin({}).string(), + "__build-remote", + }); + #if __linux__ if (isRootUser()) { try { diff --git a/src/nix/meson.build b/src/nix/meson.build index 7405548de..05bee6a87 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -33,6 +33,21 @@ subdir('build-utils-meson/threads') subdir('build-utils-meson/export-all-symbols') +configdata = configuration_data() + +fs = import('fs') + +bindir = get_option('bindir') +if not fs.is_absolute(bindir) + bindir = get_option('prefix') / bindir +endif +configdata.set_quoted('NIX_BIN_DIR', bindir) + +config_h = configure_file( + configuration : configdata, + output : 'config-nix-cli.hh', +) + add_project_arguments( # TODO(Qyriad): Yes this is how the autoconf+Make system did it. # It would be nice for our headers to be idempotent instead. @@ -42,15 +57,17 @@ add_project_arguments( #'-include', 'config-fetchers.hh', '-include', 'config-main.hh', '-include', 'config-cmd.hh', + '-include', 'config-nix-cli.hh', language : 'cpp', ) subdir('build-utils-meson/diagnostics') subdir('build-utils-meson/generate-header') -nix_sources = files( +nix_sources = [config_h] + files( 'add-to-store.cc', 'app.cc', + 'self-exe.cc', 'build.cc', 'bundle.cc', 'cat.cc', diff --git a/src/nix/repl.cc b/src/nix/repl.cc index a2f3e033e..5a570749f 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -1,12 +1,32 @@ #include "eval.hh" #include "eval-settings.hh" +#include "config-global.hh" #include "globals.hh" #include "command.hh" #include "installable-value.hh" #include "repl.hh" +#include "processes.hh" +#include "self-exe.hh" namespace nix { +void runNix(Path program, const Strings & args, + const std::optional & input = {}) +{ + auto subprocessEnv = getEnv(); + subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue(); + //isInteractive avoid grabling interactive commands + runProgram2(RunOptions { + .program = getNixBin(program).string(), + .args = args, + .environment = subprocessEnv, + .input = input, + .isInteractive = true, + }); + + return; +} + struct CmdRepl : RawInstallablesCommand { CmdRepl() { @@ -81,7 +101,8 @@ struct CmdRepl : RawInstallablesCommand lookupPath, openStore(), state, - getValues + getValues, + runNix ); repl->autoArgs = getAutoArgs(*repl->state); repl->initEnv(); diff --git a/src/nix/self-exe.cc b/src/nix/self-exe.cc new file mode 100644 index 000000000..a260bafd5 --- /dev/null +++ b/src/nix/self-exe.cc @@ -0,0 +1,38 @@ +#include "current-process.hh" +#include "file-system.hh" +#include "globals.hh" +#include "self-exe.hh" + +namespace nix { + +namespace fs = std::filesystem; + +fs::path getNixBin(std::optional binaryNameOpt) +{ + auto getBinaryName = [&] { return binaryNameOpt ? *binaryNameOpt : "nix"; }; + + // If the environment variable is set, use it unconditionally + if (auto envOpt = getEnvNonEmpty("NIX_BIN_DIR")) + return fs::path{*envOpt} / std::string{getBinaryName()}; + + // Use some-times avaiable OS tricks to get to the path of this Nix, and try that + if (auto selfOpt = getSelfExe()) { + fs::path path{*selfOpt}; + if (binaryNameOpt) + path = path.parent_path() / std::string{*binaryNameOpt}; + if (fs::exists(path)) + return path; + } + + // If `nix` exists at the hardcoded fallback path, use it. + { + auto path = fs::path{NIX_BIN_DIR} / std::string{getBinaryName()}; + if (fs::exists(path)) + return path; + } + + // return just the name, hoping the exe is on the `PATH` + return getBinaryName(); +} + +} diff --git a/src/nix/self-exe.hh b/src/nix/self-exe.hh new file mode 100644 index 000000000..0772afa67 --- /dev/null +++ b/src/nix/self-exe.hh @@ -0,0 +1,31 @@ +#pragma once +///@file + +#include + +namespace nix { + +/** + * Get a path to the given Nix binary. + * + * Normally, nix is installed according to `NIX_BIN_DIR`, which is set + * at compile time, but can be overridden. + * + * However, it may not have been installed at all. For example, if it's + * a static build, there's a good chance that it has been moved out of + * its installation directory. That makes `NIX_BIN_DIR` useless. + * Instead, we'll query the OS for the path to the current executable, + * using `getSelfExe()`. + * + * As a last resort, we resort to `PATH`. Hopefully we find a `nix` + * there that's compatible. If you're porting Nix to a new platform, + * that might be good enough for a while, but you'll want to improve + * `getSelfExe()` to work on your platform. + * + * @param binary_name the exact binary name we're looking up. Might be + * `nix-*` instead of `nix` for the legacy CLI commands. Optional to use + * current binary name. + */ +std::filesystem::path getNixBin(std::optional binary_name = {}); + +} diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 9ca3f6087..f54cc59d0 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -9,6 +9,7 @@ #include "names.hh" #include "progress-bar.hh" #include "executable-path.hh" +#include "self-exe.hh" using namespace nix; @@ -93,7 +94,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand { Activity act(*logger, lvlInfo, actUnknown, fmt("installing '%s' into profile '%s'...", store->printStorePath(storePath), profileDir)); - runProgram(settings.nixBinDir + "/nix-env", false, + runProgram(getNixBin("nix-env").string(), false, {"--profile", profileDir, "-i", store->printStorePath(storePath), "--no-sandbox"}); } @@ -103,7 +104,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand /* Return the profile in which Nix is installed. */ Path getProfileDir(ref store) { - auto whereOpt = ExecutablePath::load().find(OS_STR("nix-env")); + auto whereOpt = ExecutablePath::load().findName(OS_STR("nix-env")); if (!whereOpt) throw Error("couldn't figure out how Nix is installed, so I can't upgrade it"); auto & where = *whereOpt; diff --git a/src/perl/lib/Nix/Config.pm.in b/src/perl/lib/Nix/Config.pm.in index 508a15e15..ad51cff3b 100644 --- a/src/perl/lib/Nix/Config.pm.in +++ b/src/perl/lib/Nix/Config.pm.in @@ -5,7 +5,6 @@ use Nix::Store; $version = "@PACKAGE_VERSION@"; -$binDir = Nix::Store::getBinDir; $storeDir = Nix::Store::getStoreDir; %config = (); diff --git a/src/perl/lib/Nix/Store.pm b/src/perl/lib/Nix/Store.pm index 16f2e17c8..f2ae7e88f 100644 --- a/src/perl/lib/Nix/Store.pm +++ b/src/perl/lib/Nix/Store.pm @@ -24,7 +24,7 @@ our @EXPORT = qw( hashPath hashFile hashString convertHash signString checkSignature - getBinDir getStoreDir + getStoreDir setVerbosity ); diff --git a/src/perl/lib/Nix/Store.xs b/src/perl/lib/Nix/Store.xs index f951437c8..172c3500d 100644 --- a/src/perl/lib/Nix/Store.xs +++ b/src/perl/lib/Nix/Store.xs @@ -424,11 +424,6 @@ StoreWrapper::addTempRoot(char * storePath) } -SV * getBinDir() - PPCODE: - XPUSHs(sv_2mortal(newSVpv(settings.nixBinDir.c_str(), 0))); - - SV * getStoreDir() PPCODE: XPUSHs(sv_2mortal(newSVpv(settings.nixStore.c_str(), 0))); From f22bf867eb41a98441ab44b3667d3abb146ee603 Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Mon, 12 Aug 2024 22:18:14 -0400 Subject: [PATCH 129/142] fix: use SymbolStr in constructor --- src/libexpr/value.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 257da1d2d..39144fe15 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -325,9 +325,9 @@ public: void mkStringMove(const char * s, const NixStringContext & context); - inline void mkString(const Symbol & s) + inline void mkString(const SymbolStr & s) { - mkString(((const std::string &) s).c_str()); + mkString(s.c_str()); } void mkPath(const SourcePath & path); From 95fe9f5ba18bb5fe025a06cbba0e18740ad101dd Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 9 Jul 2024 14:46:26 -0400 Subject: [PATCH 130/142] Fix Meson installation of the Nix CLI Co-Authored-By: Qyriad --- src/nix/meson.build | 52 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/src/nix/meson.build b/src/nix/meson.build index 05bee6a87..798c98e33 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -192,3 +192,55 @@ this_exe = executable( link_args: linker_export_flags, install : true, ) + +meson.override_find_program('nix', this_exe) + +nix_symlinks = [ + 'nix-build', + 'nix-channel', + 'nix-collect-garbage', + 'nix-copy-closure', + 'nix-daemon', + 'nix-env', + 'nix-hash', + 'nix-instantiate', + 'nix-prefetch-url', + 'nix-shell', + 'nix-store', +] + +foreach linkname : nix_symlinks + install_symlink( + linkname, + # TODO(Qyriad): should these continue to be relative symlinks? + pointing_to : 'nix', + install_dir : get_option('bindir'), + # The 'runtime' tag is what executables default to, which we want to emulate here. + install_tag : 'runtime' + ) + t = custom_target( + command: ['ln', '-sf', fs.name(this_exe), '@OUTPUT@'], + output: linkname, + # TODO(Ericson2314): Don't do this once we have the `meson.override_find_program` working) + build_by_default: true + ) + # TODO(Ericson3214): Dosen't yet work + #meson.override_find_program(linkname, t) +endforeach + +install_symlink( + 'build-remote', + pointing_to : '..' / '..'/ get_option('bindir') / 'nix', + install_dir : get_option('libexecdir') / 'nix', + # The 'runtime' tag is what executables default to, which we want to emulate here. + install_tag : 'runtime' +) + +custom_target( + command: ['ln', '-sf', fs.name(this_exe), '@OUTPUT@'], + output: 'build-remote', + # TODO(Ericson2314): Don't do this once we have the `meson.override_find_program` working) + build_by_default: true +) +# TODO(Ericson3214): Dosen't yet work +#meson.override_find_program(linkname, t) From 4956e7c44c5018c2c73a125913e264558cd638d6 Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Tue, 13 Aug 2024 19:22:32 +0200 Subject: [PATCH 131/142] add cross-references to `nix-path` overriding (#11288) * add cross-references to `nix-path` overriding while this information is already present in the settings, it's more likely to be first accessed through the "lookup path" page, which currently requires following two links to get to the practically important bits. Co-authored-by: Robert Hensing --- doc/manual/src/language/constructs/lookup-path.md | 7 ++----- src/libexpr/primops.cc | 5 ++++- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/manual/src/language/constructs/lookup-path.md b/doc/manual/src/language/constructs/lookup-path.md index 11b9fe88c..a2e80280b 100644 --- a/doc/manual/src/language/constructs/lookup-path.md +++ b/doc/manual/src/language/constructs/lookup-path.md @@ -4,11 +4,8 @@ > > *lookup-path* = `<` *identifier* [ `/` *identifier* ]... `>` -A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/types.md#type-path) if the identifier matches a search path entry. - -The value of a lookup path is determined by [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath). - -See [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile) for details on lookup path resolution. +A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/types.md#type-path) if the identifier matches a search path entry in [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath). +The algorithm for lookup path resolution is described in the documentation on [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile). > **Example** > diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 2de6cbf4b..9de8ff599 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -4857,7 +4857,10 @@ void EvalState::createBaseEnv() addConstant("__nixPath", v, { .type = nList, .doc = R"( - The value of the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path): a list of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). + A list of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). + Its value is primarily determined by the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path), which are + - Overridden by the [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) environment variable or the `--nix-path` option + - Extended by the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I) or `--extra-nix-path` > **Example** > From 612fc76020495cefee6a5f5259fd83a531b0ef31 Mon Sep 17 00:00:00 2001 From: bryango Date: Wed, 14 Aug 2024 20:27:12 +0800 Subject: [PATCH 132/142] doc/manual: fix misaligned icons in custom.css (#11296) --- doc/manual/custom.css | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/custom.css b/doc/manual/custom.css index 9e8e3886f..7af150be3 100644 --- a/doc/manual/custom.css +++ b/doc/manual/custom.css @@ -12,8 +12,8 @@ h1.menu-title::before { } -h1.menu-title { - padding: 0.5em; +.menu-bar { + padding: 0.5em 0em; } .sidebar .sidebar-scrollbox { From cc9fe4dee713717c88b22c8701f9bdffc786baa1 Mon Sep 17 00:00:00 2001 From: Bryan Honof Date: Fri, 2 Aug 2024 00:10:08 +0200 Subject: [PATCH 133/142] Fix a few shellcheck tests Ref nixos/nix#10795 --- maintainers/flake-module.nix | 11 --- tests/functional/signing.sh | 94 +++++++++---------- tests/functional/simple.builder.sh | 7 +- tests/functional/simple.sh | 10 +- tests/functional/ssh-relay.sh | 8 +- tests/functional/store-info.sh | 4 +- tests/functional/structured-attrs.sh | 11 ++- .../functional/substitute-with-invalid-ca.sh | 8 +- tests/functional/suggestions.sh | 2 +- tests/functional/tarball.sh | 44 ++++----- tests/functional/test-infra.sh | 5 +- tests/functional/test-libstoreconsumer.sh | 2 +- tests/functional/timeout.sh | 2 +- 13 files changed, 102 insertions(+), 106 deletions(-) diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index c3eaf671c..0b83e5696 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -641,19 +641,8 @@ ''^tests/functional/selfref-gc\.sh$'' ''^tests/functional/shell\.sh$'' ''^tests/functional/shell\.shebang\.sh$'' - ''^tests/functional/signing\.sh$'' ''^tests/functional/simple\.builder\.sh$'' - ''^tests/functional/simple\.sh$'' - ''^tests/functional/ssh-relay\.sh$'' - ''^tests/functional/store-info\.sh$'' - ''^tests/functional/structured-attrs\.sh$'' - ''^tests/functional/substitute-with-invalid-ca\.sh$'' - ''^tests/functional/suggestions\.sh$'' ''^tests/functional/supplementary-groups\.sh$'' - ''^tests/functional/tarball\.sh$'' - ''^tests/functional/test-infra\.sh$'' - ''^tests/functional/test-libstoreconsumer\.sh$'' - ''^tests/functional/timeout\.sh$'' ''^tests/functional/toString-path\.sh$'' ''^tests/functional/user-envs-migration\.sh$'' ''^tests/functional/user-envs-test-case\.sh$'' diff --git a/tests/functional/signing.sh b/tests/functional/signing.sh index 890d1446f..8ec093a48 100755 --- a/tests/functional/signing.sh +++ b/tests/functional/signing.sh @@ -5,108 +5,108 @@ source common.sh clearStoreIfPossible clearCache -nix-store --generate-binary-cache-key cache1.example.org $TEST_ROOT/sk1 $TEST_ROOT/pk1 -pk1=$(cat $TEST_ROOT/pk1) -nix-store --generate-binary-cache-key cache2.example.org $TEST_ROOT/sk2 $TEST_ROOT/pk2 -pk2=$(cat $TEST_ROOT/pk2) +nix-store --generate-binary-cache-key cache1.example.org "$TEST_ROOT"/sk1 "$TEST_ROOT"/pk1 +pk1=$(cat "$TEST_ROOT"/pk1) +nix-store --generate-binary-cache-key cache2.example.org "$TEST_ROOT"/sk2 "$TEST_ROOT"/pk2 +pk2=$(cat "$TEST_ROOT"/pk2) # Build a path. outPath=$(nix-build dependencies.nix --no-out-link --secret-key-files "$TEST_ROOT/sk1 $TEST_ROOT/sk2") # Verify that the path got signed. -info=$(nix path-info --json $outPath) -echo $info | jq -e '.[] | .ultimate == true' +info=$(nix path-info --json "$outPath") +echo "$info" | jq -e '.[] | .ultimate == true' TODO_NixOS # looks like an actual bug? Following line fails on NixOS: -echo $info | jq -e '.[] | .signatures.[] | select(startswith("cache1.example.org"))' -echo $info | jq -e '.[] | .signatures.[] | select(startswith("cache2.example.org"))' +echo "$info" | jq -e '.[] | .signatures.[] | select(startswith("cache1.example.org"))' +echo "$info" | jq -e '.[] | .signatures.[] | select(startswith("cache2.example.org"))' # Test "nix store verify". -nix store verify -r $outPath +nix store verify -r "$outPath" -expect 2 nix store verify -r $outPath --sigs-needed 1 +expect 2 nix store verify -r "$outPath" --sigs-needed 1 -nix store verify -r $outPath --sigs-needed 1 --trusted-public-keys $pk1 +nix store verify -r "$outPath" --sigs-needed 1 --trusted-public-keys "$pk1" -expect 2 nix store verify -r $outPath --sigs-needed 2 --trusted-public-keys $pk1 +expect 2 nix store verify -r "$outPath" --sigs-needed 2 --trusted-public-keys "$pk1" -nix store verify -r $outPath --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" +nix store verify -r "$outPath" --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" nix store verify --all --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" # Build something unsigned. outPath2=$(nix-build simple.nix --no-out-link) -nix store verify -r $outPath +nix store verify -r "$outPath" # Verify that the path did not get signed but does have the ultimate bit. -info=$(nix path-info --json $outPath2) -echo $info | jq -e '.[] | .ultimate == true' -echo $info | jq -e '.[] | .signatures == []' +info=$(nix path-info --json "$outPath2") +echo "$info" | jq -e '.[] | .ultimate == true' +echo "$info" | jq -e '.[] | .signatures == []' # Test "nix store verify". -nix store verify -r $outPath2 +nix store verify -r "$outPath2" -expect 2 nix store verify -r $outPath2 --sigs-needed 1 +expect 2 nix store verify -r "$outPath2" --sigs-needed 1 -expect 2 nix store verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1 +expect 2 nix store verify -r "$outPath2" --sigs-needed 1 --trusted-public-keys "$pk1" # Test "nix store sign". -nix store sign --key-file $TEST_ROOT/sk1 $outPath2 +nix store sign --key-file "$TEST_ROOT"/sk1 "$outPath2" -nix store verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1 +nix store verify -r "$outPath2" --sigs-needed 1 --trusted-public-keys "$pk1" # Build something content-addressed. outPathCA=$(IMPURE_VAR1=foo IMPURE_VAR2=bar nix-build ./fixed.nix -A good.0 --no-out-link) -nix path-info --json $outPathCA | jq -e '.[] | .ca | startswith("fixed:md5:")' +nix path-info --json "$outPathCA" | jq -e '.[] | .ca | startswith("fixed:md5:")' # Content-addressed paths don't need signatures, so they verify # regardless of --sigs-needed. -nix store verify $outPathCA -nix store verify $outPathCA --sigs-needed 1000 +nix store verify "$outPathCA" +nix store verify "$outPathCA" --sigs-needed 1000 # Check that signing a content-addressed path doesn't overflow validSigs -nix store sign --key-file $TEST_ROOT/sk1 $outPathCA -nix store verify -r $outPathCA --sigs-needed 1000 --trusted-public-keys $pk1 +nix store sign --key-file "$TEST_ROOT"/sk1 "$outPathCA" +nix store verify -r "$outPathCA" --sigs-needed 1000 --trusted-public-keys "$pk1" # Copy to a binary cache. -nix copy --to file://$cacheDir $outPath2 +nix copy --to file://"$cacheDir" "$outPath2" # Verify that signatures got copied. -info=$(nix path-info --store file://$cacheDir --json $outPath2) -echo $info | jq -e '.[] | .ultimate == false' -echo $info | jq -e '.[] | .signatures.[] | select(startswith("cache1.example.org"))' -echo $info | expect 4 jq -e '.[] | .signatures.[] | select(startswith("cache2.example.org"))' +info=$(nix path-info --store file://"$cacheDir" --json "$outPath2") +echo "$info" | jq -e '.[] | .ultimate == false' +echo "$info" | jq -e '.[] | .signatures.[] | select(startswith("cache1.example.org"))' +echo "$info" | expect 4 jq -e '.[] | .signatures.[] | select(startswith("cache2.example.org"))' # Verify that adding a signature to a path in a binary cache works. -nix store sign --store file://$cacheDir --key-file $TEST_ROOT/sk2 $outPath2 -info=$(nix path-info --store file://$cacheDir --json $outPath2) -echo $info | jq -e '.[] | .signatures.[] | select(startswith("cache1.example.org"))' -echo $info | jq -e '.[] | .signatures.[] | select(startswith("cache2.example.org"))' +nix store sign --store file://"$cacheDir" --key-file "$TEST_ROOT"/sk2 "$outPath2" +info=$(nix path-info --store file://"$cacheDir" --json "$outPath2") +echo "$info" | jq -e '.[] | .signatures.[] | select(startswith("cache1.example.org"))' +echo "$info" | jq -e '.[] | .signatures.[] | select(startswith("cache2.example.org"))' # Copying to a diverted store should fail due to a lack of signatures by trusted keys. -chmod -R u+w $TEST_ROOT/store0 || true -rm -rf $TEST_ROOT/store0 +chmod -R u+w "$TEST_ROOT"/store0 || true +rm -rf "$TEST_ROOT"/store0 # Fails or very flaky only on GHA + macOS: # expectStderr 1 nix copy --to $TEST_ROOT/store0 $outPath | grepQuiet -E 'cannot add path .* because it lacks a signature by a trusted key' # but this works: -(! nix copy --to $TEST_ROOT/store0 $outPath) +(! nix copy --to "$TEST_ROOT"/store0 "$outPath") # But succeed if we supply the public keys. -nix copy --to $TEST_ROOT/store0 $outPath --trusted-public-keys $pk1 +nix copy --to "$TEST_ROOT"/store0 "$outPath" --trusted-public-keys "$pk1" -expect 2 nix store verify --store $TEST_ROOT/store0 -r $outPath +expect 2 nix store verify --store "$TEST_ROOT"/store0 -r "$outPath" -nix store verify --store $TEST_ROOT/store0 -r $outPath --trusted-public-keys $pk1 -nix store verify --store $TEST_ROOT/store0 -r $outPath --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" +nix store verify --store "$TEST_ROOT"/store0 -r "$outPath" --trusted-public-keys "$pk1" +nix store verify --store "$TEST_ROOT"/store0 -r "$outPath" --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" # It should also succeed if we disable signature checking. -(! nix copy --to $TEST_ROOT/store0 $outPath2) -nix copy --to $TEST_ROOT/store0?require-sigs=false $outPath2 +(! nix copy --to "$TEST_ROOT"/store0 "$outPath2") +nix copy --to "$TEST_ROOT"/store0?require-sigs=false "$outPath2" # But signatures should still get copied. -nix store verify --store $TEST_ROOT/store0 -r $outPath2 --trusted-public-keys $pk1 +nix store verify --store "$TEST_ROOT"/store0 -r "$outPath2" --trusted-public-keys "$pk1" # Content-addressed stuff can be copied without signatures. -nix copy --to $TEST_ROOT/store0 $outPathCA +nix copy --to "$TEST_ROOT"/store0 "$outPathCA" diff --git a/tests/functional/simple.builder.sh b/tests/functional/simple.builder.sh index 569e8ca88..97abf0676 100644 --- a/tests/functional/simple.builder.sh +++ b/tests/functional/simple.builder.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + echo "PATH=$PATH" # Verify that the PATH is empty. @@ -5,7 +7,6 @@ if mkdir foo 2> /dev/null; then exit 1; fi # Set a PATH (!!! impure). export PATH=$goodPath +mkdir "$out" -mkdir $out - -echo "Hello World!" > $out/hello \ No newline at end of file +echo "Hello World!" > "$out"/hello diff --git a/tests/functional/simple.sh b/tests/functional/simple.sh index 86acca0c2..8afa369c2 100755 --- a/tests/functional/simple.sh +++ b/tests/functional/simple.sh @@ -12,7 +12,7 @@ outPath=$(nix-store -rvv "$drvPath") echo "output path is $outPath" -(! [ -w $outPath ]) +[[ ! -w $outPath ]] text=$(cat "$outPath/hello") if test "$text" != "Hello World!"; then exit 1; fi @@ -21,16 +21,16 @@ TODO_NixOS # Directed delete: $outPath is not reachable from a root, so it should # be deleteable. -nix-store --delete $outPath -(! [ -e $outPath/hello ]) +nix-store --delete "$outPath" +[[ ! -e $outPath/hello ]] -outPath="$(NIX_REMOTE=local?store=/foo\&real=$TEST_ROOT/real-store nix-instantiate --readonly-mode hash-check.nix)" +outPath="$(NIX_REMOTE='local?store=/foo&real='"$TEST_ROOT"'/real-store' nix-instantiate --readonly-mode hash-check.nix)" if test "$outPath" != "/foo/lfy1s6ca46rm5r6w4gg9hc0axiakjcnm-dependencies.drv"; then echo "hashDerivationModulo appears broken, got $outPath" exit 1 fi -outPath="$(NIX_REMOTE=local?store=/foo\&real=$TEST_ROOT/real-store nix-instantiate --readonly-mode big-derivation-attr.nix)" +outPath="$(NIX_REMOTE='local?store=/foo&real='"$TEST_ROOT"'/real-store' nix-instantiate --readonly-mode big-derivation-attr.nix)" if test "$outPath" != "/foo/xxiwa5zlaajv6xdjynf9yym9g319d6mn-big-derivation-attr.drv"; then echo "big-derivation-attr.nix hash appears broken, got $outPath. Memory corruption in large drv attr?" exit 1 diff --git a/tests/functional/ssh-relay.sh b/tests/functional/ssh-relay.sh index 059c66434..71b8ae9ab 100755 --- a/tests/functional/ssh-relay.sh +++ b/tests/functional/ssh-relay.sh @@ -2,10 +2,10 @@ source common.sh -echo foo > $TEST_ROOT/hello.sh +echo foo > "$TEST_ROOT"/hello.sh ssh_localhost=ssh://localhost -remote_store=?remote-store=$ssh_localhost +remote_store="?remote-store=$ssh_localhost" store=$ssh_localhost @@ -13,6 +13,6 @@ store+=$remote_store store+=$remote_store store+=$remote_store -out=$(nix store add-path --store "$store" $TEST_ROOT/hello.sh) +out=$(nix store add-path --store "$store" "$TEST_ROOT"/hello.sh) -[ foo = $(< $out) ] +[ foo = "$(< "$out")" ] diff --git a/tests/functional/store-info.sh b/tests/functional/store-info.sh index f37889fbb..beecc2dd9 100755 --- a/tests/functional/store-info.sh +++ b/tests/functional/store-info.sh @@ -8,12 +8,12 @@ STORE_INFO_JSON=$(nix store info --json) echo "$STORE_INFO" | grep "Store URL: ${NIX_REMOTE}" if [[ -v NIX_DAEMON_PACKAGE ]] && isDaemonNewer "2.7.0pre20220126"; then - DAEMON_VERSION=$($NIX_DAEMON_PACKAGE/bin/nix daemon --version | cut -d' ' -f3) + DAEMON_VERSION=$("$NIX_DAEMON_PACKAGE"/bin/nix daemon --version | cut -d' ' -f3) echo "$STORE_INFO" | grep "Version: $DAEMON_VERSION" [[ "$(echo "$STORE_INFO_JSON" | jq -r ".version")" == "$DAEMON_VERSION" ]] fi -expect 127 NIX_REMOTE=unix:$PWD/store nix store info || \ +expect 127 NIX_REMOTE=unix:"$PWD"/store nix store info || \ fail "nix store info on a non-existent store should fail" TODO_NixOS diff --git a/tests/functional/structured-attrs.sh b/tests/functional/structured-attrs.sh index ec1282668..64d136e99 100755 --- a/tests/functional/structured-attrs.sh +++ b/tests/functional/structured-attrs.sh @@ -8,17 +8,19 @@ requireDaemonNewerThan "2.4pre20210712" clearStoreIfPossible -rm -f $TEST_ROOT/result +rm -f "$TEST_ROOT"/result -nix-build structured-attrs.nix -A all -o $TEST_ROOT/result +nix-build structured-attrs.nix -A all -o "$TEST_ROOT"/result -[[ $(cat $TEST_ROOT/result/foo) = bar ]] -[[ $(cat $TEST_ROOT/result-dev/foo) = foo ]] +[[ $(cat "$TEST_ROOT"/result/foo) = bar ]] +[[ $(cat "$TEST_ROOT"/result-dev/foo) = foo ]] export NIX_BUILD_SHELL=$SHELL +# shellcheck disable=SC2016 env NIX_PATH=nixpkgs=shell.nix nix-shell structured-attrs-shell.nix \ --run 'test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"' +# shellcheck disable=SC2016 nix develop -f structured-attrs-shell.nix -c bash -c 'test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"' TODO_NixOS # following line fails. @@ -26,6 +28,7 @@ TODO_NixOS # following line fails. # `nix develop` is a slightly special way of dealing with environment vars, it parses # these from a shell-file exported from a derivation. This is to test especially `outputs` # (which is an associative array in thsi case) being fine. +# shellcheck disable=SC2016 nix develop -f structured-attrs-shell.nix -c bash -c 'test -n "$out"' nix print-dev-env -f structured-attrs-shell.nix | grepQuiet 'NIX_ATTRS_JSON_FILE=' diff --git a/tests/functional/substitute-with-invalid-ca.sh b/tests/functional/substitute-with-invalid-ca.sh index d8af67237..33432e95d 100755 --- a/tests/functional/substitute-with-invalid-ca.sh +++ b/tests/functional/substitute-with-invalid-ca.sh @@ -11,16 +11,16 @@ getRemoteNarInfo () { echo "$cacheDir/$(getHash "$1").narinfo" } -cat < $TEST_HOME/good.txt +cat < "$TEST_HOME"/good.txt I’m a good path EOF -cat < $TEST_HOME/bad.txt +cat < "$TEST_HOME"/bad.txt I’m a bad path EOF -good=$(nix-store --add $TEST_HOME/good.txt) -bad=$(nix-store --add $TEST_HOME/bad.txt) +good=$(nix-store --add "$TEST_HOME"/good.txt) +bad=$(nix-store --add "$TEST_HOME"/bad.txt) nix copy --to "$BINARY_CACHE" "$good" nix copy --to "$BINARY_CACHE" "$bad" nix-collect-garbage >/dev/null 2>&1 diff --git a/tests/functional/suggestions.sh b/tests/functional/suggestions.sh index 8db6f7b97..fbca93da8 100755 --- a/tests/functional/suggestions.sh +++ b/tests/functional/suggestions.sh @@ -37,7 +37,7 @@ NIX_BUILD_STDERR_WITH_NO_CLOSE_SUGGESTION=$(! nix build .\#bar 2>&1 1>/dev/null) [[ ! "$NIX_BUILD_STDERR_WITH_NO_CLOSE_SUGGESTION" =~ "Did you mean" ]] || \ fail "The nix build stderr shouldn’t suggest anything if there’s nothing relevant to suggest" -NIX_EVAL_STDERR_WITH_SUGGESTIONS=$(! nix build --impure --expr '(builtins.getFlake (builtins.toPath ./.)).packages.'$system'.fob' 2>&1 1>/dev/null) +NIX_EVAL_STDERR_WITH_SUGGESTIONS=$(! nix build --impure --expr '(builtins.getFlake (builtins.toPath ./.)).packages.'"$system"'.fob' 2>&1 1>/dev/null) [[ "$NIX_EVAL_STDERR_WITH_SUGGESTIONS" =~ "Did you mean one of fo1, fo2, foo or fooo?" ]] || \ fail "The evaluator should suggest the three closest possiblities" diff --git a/tests/functional/tarball.sh b/tests/functional/tarball.sh index 4d8945625..dee0a98f1 100755 --- a/tests/functional/tarball.sh +++ b/tests/functional/tarball.sh @@ -4,51 +4,51 @@ source common.sh clearStoreIfPossible -rm -rf $TEST_HOME +rm -rf "$TEST_HOME" tarroot=$TEST_ROOT/tarball -rm -rf $tarroot -mkdir -p $tarroot -cp dependencies.nix $tarroot/default.nix -cp config.nix dependencies.builder*.sh $tarroot/ -touch -d '@1000000000' $tarroot $tarroot/* +rm -rf "$tarroot" +mkdir -p "$tarroot" +cp dependencies.nix "$tarroot/default.nix" +cp config.nix dependencies.builder*.sh "$tarroot/" +touch -d '@1000000000' "$tarroot" "$tarroot"/* -hash=$(nix hash path $tarroot) +hash=$(nix hash path "$tarroot") test_tarball() { local ext="$1" local compressor="$2" tarball=$TEST_ROOT/tarball.tar$ext - (cd $TEST_ROOT && GNUTAR_REPRODUCIBLE= tar --mtime=$tarroot/default.nix --owner=0 --group=0 --numeric-owner --sort=name -c -f - tarball) | $compressor > $tarball + (cd "$TEST_ROOT" && GNUTAR_REPRODUCIBLE=1 tar --mtime="$tarroot"/default.nix --owner=0 --group=0 --numeric-owner --sort=name -c -f - tarball) | $compressor > "$tarball" - nix-env -f file://$tarball -qa --out-path | grepQuiet dependencies + nix-env -f file://"$tarball" -qa --out-path | grepQuiet dependencies - nix-build -o $TEST_ROOT/result file://$tarball + nix-build -o "$TEST_ROOT"/result file://"$tarball" - nix-build -o $TEST_ROOT/result '' -I foo=file://$tarball + nix-build -o "$TEST_ROOT"/result '' -I foo=file://"$tarball" - nix-build -o $TEST_ROOT/result -E "import (fetchTarball file://$tarball)" + nix-build -o "$TEST_ROOT"/result -E "import (fetchTarball file://$tarball)" # Do not re-fetch paths already present - nix-build -o $TEST_ROOT/result -E "import (fetchTarball { url = file:///does-not-exist/must-remain-unused/$tarball; sha256 = \"$hash\"; })" + nix-build -o "$TEST_ROOT"/result -E "import (fetchTarball { url = file:///does-not-exist/must-remain-unused/$tarball; sha256 = \"$hash\"; })" - nix-build -o $TEST_ROOT/result -E "import (fetchTree file://$tarball)" - nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })" - nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })" + nix-build -o "$TEST_ROOT"/result -E "import (fetchTree file://$tarball)" + nix-build -o "$TEST_ROOT"/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })" + nix-build -o "$TEST_ROOT"/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })" # Do not re-fetch paths already present - nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file:///does-not-exist/must-remain-unused/$tarball; narHash = \"$hash\"; })" - expectStderr 102 nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" | grep 'NAR hash mismatch in input' + nix-build -o "$TEST_ROOT"/result -E "import (fetchTree { type = \"tarball\"; url = file:///does-not-exist/must-remain-unused/$tarball; narHash = \"$hash\"; })" + expectStderr 102 nix-build -o "$TEST_ROOT"/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" | grep 'NAR hash mismatch in input' [[ $(nix eval --impure --expr "(fetchTree file://$tarball).lastModified") = 1000000000 ]] nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" >&2 nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" 2>&1 | grep 'true' - nix-instantiate --eval -E '1 + 2' -I fnord=file:///no-such-tarball.tar$ext - nix-instantiate --eval -E 'with ; 1 + 2' -I fnord=file:///no-such-tarball$ext - (! nix-instantiate --eval -E ' 1' -I fnord=file:///no-such-tarball$ext) + nix-instantiate --eval -E '1 + 2' -I fnord=file:///no-such-tarball.tar"$ext" + nix-instantiate --eval -E 'with ; 1 + 2' -I fnord=file:///no-such-tarball"$ext" + (! nix-instantiate --eval -E ' 1' -I fnord=file:///no-such-tarball"$ext") - nix-instantiate --eval -E '' -I fnord=file:///no-such-tarball$ext -I fnord=. + nix-instantiate --eval -E '' -I fnord=file:///no-such-tarball"$ext" -I fnord=. # Ensure that the `name` attribute isn’t accepted as that would mess # with the content-addressing diff --git a/tests/functional/test-infra.sh b/tests/functional/test-infra.sh index d02a11b46..2da26b08c 100755 --- a/tests/functional/test-infra.sh +++ b/tests/functional/test-infra.sh @@ -14,7 +14,7 @@ expect 1 false expect 1 expect 0 false function ret() { - return $1 + return "$1" } # `expect` can call functions, not just executables @@ -48,6 +48,7 @@ expectStderr 1 noisyFalse | grepQuiet NAY # `set -o pipefile` is enabled +# shellcheck disable=SC2317# shellcheck disable=SC2317 pipefailure () { # shellcheck disable=SC2216 true | false | true @@ -55,6 +56,7 @@ pipefailure () { expect 1 pipefailure unset pipefailure +# shellcheck disable=SC2317 pipefailure () { # shellcheck disable=SC2216 false | true | true @@ -82,6 +84,7 @@ expect 1 useUnbound # ! alone unfortunately negates `set -e`, but it works in functions: # shellcheck disable=SC2251 ! true +# shellcheck disable=SC2317 funBang () { ! true } diff --git a/tests/functional/test-libstoreconsumer.sh b/tests/functional/test-libstoreconsumer.sh index d1a1accb6..2adead1c0 100755 --- a/tests/functional/test-libstoreconsumer.sh +++ b/tests/functional/test-libstoreconsumer.sh @@ -5,4 +5,4 @@ source common.sh drv="$(nix-instantiate simple.nix)" cat "$drv" out="$(./test-libstoreconsumer/test-libstoreconsumer "$drv")" -cat "$out/hello" | grep -F "Hello World!" +grep -F "Hello World!" < "$out/hello" diff --git a/tests/functional/timeout.sh b/tests/functional/timeout.sh index f42354538..ae47fdc96 100755 --- a/tests/functional/timeout.sh +++ b/tests/functional/timeout.sh @@ -9,7 +9,7 @@ needLocalStore "see #4813" messages=$(nix-build -Q timeout.nix -A infiniteLoop --timeout 2 2>&1) && status=0 || status=$? -if [ $status -ne 101 ]; then +if [ "$status" -ne 101 ]; then echo "error: 'nix-store' exited with '$status'; should have exited 101" # FIXME: https://github.com/NixOS/nix/issues/4813 From 93f58150c9cd5f2fc9345c2730ce8a6da47dc474 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 13 Aug 2024 16:15:56 -0400 Subject: [PATCH 134/142] Coarse versions for constituent packages As discussed in our meeting, we should use a simplified version for the libraries without the date or commit hash. This will make rebuilding a lot faster in many cases. Progress on #10379 Co-Authored-By: Robert Hensing --- flake.nix | 19 +++++++++---------- package.nix | 5 ++--- packaging/components.nix | 31 +++++++++++++++++++++++++++---- packaging/dependencies.nix | 5 +---- packaging/hydra.nix | 9 ++++++++- 5 files changed, 47 insertions(+), 22 deletions(-) diff --git a/flake.nix b/flake.nix index 9e8592e3a..638f6b4bd 100644 --- a/flake.nix +++ b/flake.nix @@ -26,12 +26,6 @@ officialRelease = false; - version = lib.fileContents ./.version + versionSuffix; - versionSuffix = - if officialRelease - then "" - else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}"; - linux32BitSystems = [ "i686-linux" ]; linux64BitSystems = [ "x86_64-linux" "aarch64-linux" ]; linuxSystems = linux32BitSystems ++ linux64BitSystems; @@ -130,12 +124,16 @@ # without "polluting" the top level "`pkgs`" attrset. # This also has the benefit of providing us with a distinct set of packages # we can iterate over. - nixComponents = lib.makeScope final.nixDependencies.newScope (import ./packaging/components.nix); + nixComponents = lib.makeScope final.nixDependencies.newScope (import ./packaging/components.nix { + inherit (final) lib; + inherit officialRelease; + src = self; + }); # The dependencies are in their own scope, so that they don't have to be # in Nixpkgs top level `pkgs` or `nixComponents`. nixDependencies = lib.makeScope final.newScope (import ./packaging/dependencies.nix { - inherit inputs stdenv versionSuffix; + inherit inputs stdenv; pkgs = final; }); @@ -170,6 +168,7 @@ linux64BitSystems nixpkgsFor self + officialRelease ; }; @@ -253,10 +252,10 @@ dockerImage = let pkgs = nixpkgsFor.${system}.native; - image = import ./docker.nix { inherit pkgs; tag = version; }; + image = import ./docker.nix { inherit pkgs; tag = pkgs.nix.version; }; in pkgs.runCommand - "docker-image-tarball-${version}" + "docker-image-tarball-${pkgs.nix.version}" { meta.description = "Docker image with Nix for ${system}"; } '' mkdir -p $out/nix-support diff --git a/package.nix b/package.nix index a7c8923e8..d41748b7c 100644 --- a/package.nix +++ b/package.nix @@ -47,7 +47,8 @@ , pname ? "nix" -, versionSuffix ? "" +, version +, versionSuffix # Whether to build Nix. Useful to skip for tasks like testing existing pre-built versions of Nix , doBuild ? true @@ -112,8 +113,6 @@ let inherit (lib) fileset; - version = lib.fileContents ./.version + versionSuffix; - # selected attributes with defaults, will be used to define some # things which should instead be gotten via `finalAttrs` in order to # work with overriding. diff --git a/packaging/components.nix b/packaging/components.nix index 870e9ae61..0e8334d10 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -1,11 +1,34 @@ +{ + lib, + src, + officialRelease, +}: + scope: + let inherit (scope) callPackage; + + baseVersion = lib.fileContents ../.version; + + versionSuffix = lib.optionalString (!officialRelease) "pre"; + + fineVersionSuffix = lib.optionalString + (!officialRelease) + "pre${builtins.substring 0 8 (src.lastModifiedDate or src.lastModified or "19700101")}_${src.shortRev or "dirty"}"; + + fineVersion = baseVersion + fineVersionSuffix; in # This becomes the pkgs.nixComponents attribute set { - nix = callPackage ../package.nix { }; + version = baseVersion + versionSuffix; + inherit versionSuffix; + + nix = callPackage ../package.nix { + version = fineVersion; + versionSuffix = fineVersionSuffix; + }; nix-util = callPackage ../src/libutil/package.nix { }; nix-util-c = callPackage ../src/libutil-c/package.nix { }; @@ -34,10 +57,10 @@ in nix-cmd = callPackage ../src/libcmd/package.nix { }; # Will replace `nix` once the old build system is gone. - nix-ng = callPackage ../src/nix/package.nix { }; + nix-ng = callPackage ../src/nix/package.nix { version = fineVersion; }; - nix-internal-api-docs = callPackage ../src/internal-api-docs/package.nix { }; - nix-external-api-docs = callPackage ../src/external-api-docs/package.nix { }; + nix-internal-api-docs = callPackage ../src/internal-api-docs/package.nix { version = fineVersion; }; + nix-external-api-docs = callPackage ../src/external-api-docs/package.nix { version = fineVersion; }; nix-perl-bindings = callPackage ../src/perl/package.nix { }; } diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index e0737593f..21c48e5cc 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -8,7 +8,6 @@ pkgs, stdenv, - versionSuffix, }: let @@ -73,11 +72,9 @@ let strictDeps = prevAttrs.strictDeps or true; enableParallelBuilding = true; }; - in scope: { - inherit stdenv versionSuffix; - version = lib.fileContents ../.version + versionSuffix; + inherit stdenv; aws-sdk-cpp = (pkgs.aws-sdk-cpp.override { apis = [ "s3" "transfer" ]; diff --git a/packaging/hydra.nix b/packaging/hydra.nix index dbe992476..9752d90e3 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -6,6 +6,7 @@ , linux64BitSystems , nixpkgsFor , self +, officialRelease }: let inherit (inputs) nixpkgs nixpkgs-regression; @@ -16,7 +17,7 @@ let }; testNixVersions = pkgs: client: daemon: - pkgs.callPackage ../package.nix { + pkgs.nixComponents.callPackage ../package.nix { pname = "nix-tests" + lib.optionalString @@ -28,6 +29,12 @@ let test-daemon = daemon; doBuild = false; + + # This could be more accurate, but a shorter version will match the + # fine version with rev. This functionality is already covered in + # the normal test, so it's fine. + version = pkgs.nixComponents.version; + versionSuffix = pkgs.nixComponents.versionSuffix; }; # Technically we could just return `pkgs.nixComponents`, but for Hydra it's From 34fe2478a2b26569e5e0b244e927d3b50da0056d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 8 Jul 2024 16:07:06 -0400 Subject: [PATCH 135/142] Build Functional tests with Meson Co-Authored-By: Qyriad Co-authored-by: Robert Hensing --- flake.nix | 4 + meson.build | 1 + packaging/components.nix | 2 + packaging/hydra.nix | 7 +- src/libstore/meson.build | 2 +- src/nix-functional-tests | 1 + tests/functional/.version | 1 + tests/functional/ca/meson.build | 33 +++ tests/functional/common.sh | 3 +- .../{vars-and-functions.sh => functions.sh} | 107 ++----- tests/functional/common/init.sh | 4 +- tests/functional/common/meson.build | 5 + tests/functional/common/paths.sh | 19 +- tests/functional/common/subst-vars.sh.in | 20 +- tests/functional/common/vars.sh | 72 +++++ .../derivation-advanced-attributes.sh | 2 +- tests/functional/dyn-drv/meson.build | 19 ++ tests/functional/flakes/meson.build | 28 ++ tests/functional/git-hashing/meson.build | 8 + .../functional/local-overlay-store/common.sh | 3 +- .../local-overlay-store/meson.build | 18 ++ tests/functional/meson.build | 266 ++++++++++++++++++ tests/functional/nested-sandboxing/command.sh | 2 + tests/functional/nested-sandboxing/runner.nix | 7 +- tests/functional/package.nix | 117 ++++++++ tests/functional/plugins.sh | 9 +- tests/functional/plugins/meson.build | 16 ++ tests/functional/restricted.sh | 3 - .../test-libstoreconsumer/meson.build | 14 + 29 files changed, 678 insertions(+), 115 deletions(-) create mode 120000 src/nix-functional-tests create mode 120000 tests/functional/.version create mode 100644 tests/functional/ca/meson.build rename tests/functional/common/{vars-and-functions.sh => functions.sh} (76%) create mode 100644 tests/functional/common/meson.build create mode 100644 tests/functional/common/vars.sh create mode 100644 tests/functional/dyn-drv/meson.build create mode 100644 tests/functional/flakes/meson.build create mode 100644 tests/functional/git-hashing/meson.build create mode 100644 tests/functional/local-overlay-store/meson.build create mode 100644 tests/functional/meson.build create mode 100644 tests/functional/package.nix create mode 100644 tests/functional/plugins/meson.build create mode 100644 tests/functional/test-libstoreconsumer/meson.build diff --git a/flake.nix b/flake.nix index 638f6b4bd..22ca54118 100644 --- a/flake.nix +++ b/flake.nix @@ -210,6 +210,9 @@ "${nixpkgsPrefix}${pkgName}-${testName}" = test; }) ) + // lib.optionalAttrs (nixpkgs.stdenv.hostPlatform == nixpkgs.stdenv.buildPlatform) { + "${nixpkgsPrefix}nix-functional-tests" = nixpkgs.nixComponents.nix-functional-tests; + } ) // devFlake.checks.${system} or {} ); @@ -323,6 +326,7 @@ ++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.nativeBuildInputs ++ pkgs.nixComponents.nix-internal-api-docs.nativeBuildInputs ++ pkgs.nixComponents.nix-external-api-docs.nativeBuildInputs + ++ pkgs.nixComponents.nix-functional-tests.baseNativeBuildInputs ++ lib.optional (!stdenv.buildPlatform.canExecute stdenv.hostPlatform # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479 diff --git a/meson.build b/meson.build index 1554244ab..715a3862d 100644 --- a/meson.build +++ b/meson.build @@ -42,3 +42,4 @@ subproject('nix-fetchers-tests') subproject('nix-expr-test-support') subproject('nix-expr-tests') subproject('nix-flake-tests') +subproject('nix-functional-tests') diff --git a/packaging/components.nix b/packaging/components.nix index 0e8334d10..f14613e8a 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -59,6 +59,8 @@ in # Will replace `nix` once the old build system is gone. nix-ng = callPackage ../src/nix/package.nix { version = fineVersion; }; + nix-functional-tests = callPackage ../src/nix-functional-tests/package.nix { version = fineVersion; }; + nix-internal-api-docs = callPackage ../src/internal-api-docs/package.nix { version = fineVersion; }; nix-external-api-docs = callPackage ../src/external-api-docs/package.nix { version = fineVersion; }; diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 9752d90e3..46b4ff51d 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -62,6 +62,7 @@ let "nix-main-c" "nix-cmd" "nix-ng" + "nix-functional-tests" ]; in { @@ -75,8 +76,10 @@ in lib.genAttrs linux64BitSystems (system: nixpkgsFor.${system}.static.nixComponents.${pkgName})); buildCross = forAllPackages (pkgName: - forAllCrossSystems (crossSystem: - lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}))); + # Hack to avoid non-evaling package + (if pkgName == "nix-functional-tests" then lib.flip builtins.removeAttrs ["x86_64-w64-mingw32"] else lib.id) + (forAllCrossSystems (crossSystem: + lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName})))); buildNoGc = forAllSystems (system: self.packages.${system}.nix.override { enableGC = false; } diff --git a/src/libstore/meson.build b/src/libstore/meson.build index d2cc235fd..8e30845e1 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -21,7 +21,7 @@ configdata = configuration_data() # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) -configdata.set_quoted('SYSTEM', host_machine.system()) +configdata.set_quoted('SYSTEM', host_machine.cpu_family() + '-' + host_machine.system()) deps_private_maybe_subproject = [ ] diff --git a/src/nix-functional-tests b/src/nix-functional-tests new file mode 120000 index 000000000..ed0cdf60b --- /dev/null +++ b/src/nix-functional-tests @@ -0,0 +1 @@ +../tests/functional \ No newline at end of file diff --git a/tests/functional/.version b/tests/functional/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/tests/functional/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/tests/functional/ca/meson.build b/tests/functional/ca/meson.build new file mode 100644 index 000000000..f682ab28f --- /dev/null +++ b/tests/functional/ca/meson.build @@ -0,0 +1,33 @@ +configure_file( + input : 'config.nix.in', + output : 'config.nix', + configuration : test_confdata, +) + +suites += { + 'name': 'ca', + 'deps': [], + 'tests': [ + 'build-with-garbage-path.sh', + 'build.sh', + 'build-cache.sh', + 'concurrent-builds.sh', + 'derivation-json.sh', + 'duplicate-realisation-in-closure.sh', + 'eval-store.sh', + 'gc.sh', + 'import-derivation.sh', + 'new-build-cmd.sh', + 'nix-copy.sh', + 'nix-run.sh', + 'nix-shell.sh', + 'post-hook.sh', + 'recursive.sh', + 'repl.sh', + 'selfref-gc.sh', + 'signatures.sh', + 'substitute.sh', + 'why-depends.sh', + ], + 'workdir': meson.current_build_dir(), +} diff --git a/tests/functional/common.sh b/tests/functional/common.sh index d038aaf59..325fac44c 100644 --- a/tests/functional/common.sh +++ b/tests/functional/common.sh @@ -8,7 +8,8 @@ COMMON_SH_SOURCED=1 functionalTestsDir="$(readlink -f "$(dirname "${BASH_SOURCE[0]-$0}")")" -source "$functionalTestsDir/common/vars-and-functions.sh" +source "$functionalTestsDir/common/vars.sh" +source "$functionalTestsDir/common/functions.sh" source "$functionalTestsDir/common/init.sh" if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then diff --git a/tests/functional/common/vars-and-functions.sh b/tests/functional/common/functions.sh similarity index 76% rename from tests/functional/common/vars-and-functions.sh rename to tests/functional/common/functions.sh index 632c81a82..d05fac4e7 100644 --- a/tests/functional/common/vars-and-functions.sh +++ b/tests/functional/common/functions.sh @@ -1,10 +1,10 @@ -# NOTE: instances of @variable@ are substituted as defined in /mk/templates.mk +# shellcheck shell=bash set -eu -o pipefail -if [[ -z "${COMMON_VARS_AND_FUNCTIONS_SH_SOURCED-}" ]]; then +if [[ -z "${COMMON_FUNCTIONS_SH_SOURCED-}" ]]; then -COMMON_VARS_AND_FUNCTIONS_SH_SOURCED=1 +COMMON_FUNCTIONS_SH_SOURCED=1 isTestOnNixOS() { [[ "${isTestOnNixOS:-}" == 1 ]] @@ -15,64 +15,14 @@ die() { exit 1 } -set +x - -commonDir="$(readlink -f "$(dirname "${BASH_SOURCE[0]-$0}")")" - -source "$commonDir/subst-vars.sh" -# Make sure shellcheck knows all these will be defined by the above generated snippet -: "${bindir?} ${coreutils?} ${dot?} ${SHELL?} ${PAGER?} ${busybox?} ${version?} ${system?} ${BUILD_SHARED_LIBS?}" - -source "$commonDir/paths.sh" -source "$commonDir/test-root.sh" - -test_nix_conf_dir=$TEST_ROOT/etc -test_nix_conf=$test_nix_conf_dir/nix.conf - -export TEST_HOME=$TEST_ROOT/test-home - -if ! isTestOnNixOS; then - export NIX_STORE_DIR - if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then - # Maybe the build directory is symlinked. - export NIX_IGNORE_SYMLINK_STORE=1 - NIX_STORE_DIR=$TEST_ROOT/store - fi - export NIX_LOCALSTATE_DIR=$TEST_ROOT/var - export NIX_LOG_DIR=$TEST_ROOT/var/log/nix - export NIX_STATE_DIR=$TEST_ROOT/var/nix - export NIX_CONF_DIR=$test_nix_conf_dir - export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/dSocket - unset NIX_USER_CONF_FILES - export _NIX_TEST_SHARED=$TEST_ROOT/shared - if [[ -n $NIX_STORE ]]; then - export _NIX_TEST_NO_SANDBOX=1 - fi - export _NIX_IN_TEST=$TEST_ROOT/shared - export _NIX_TEST_NO_LSOF=1 - export NIX_REMOTE=${NIX_REMOTE_-} - -fi # ! isTestOnNixOS - -unset NIX_PATH -export HOME=$TEST_HOME -unset XDG_STATE_HOME -unset XDG_DATA_HOME -unset XDG_CONFIG_HOME -unset XDG_CONFIG_DIRS -unset XDG_CACHE_HOME - -export IMPURE_VAR1=foo -export IMPURE_VAR2=bar - -cacheDir=$TEST_ROOT/binary-cache - readLink() { + # TODO fix this + # shellcheck disable=SC2012 ls -l "$1" | sed 's/.*->\ //' } clearProfiles() { - profiles="$HOME"/.local/state/nix/profiles + profiles="$HOME/.local/state/nix/profiles" rm -rf "$profiles" } @@ -105,11 +55,11 @@ doClearStore() { } clearCache() { - rm -rf "$cacheDir" + rm -rf "${cacheDir?}" } clearCacheCache() { - rm -f $TEST_HOME/.cache/nix/binary-cache* + rm -f "$TEST_HOME/.cache/nix/binary-cache"* } startDaemon() { @@ -122,7 +72,7 @@ startDaemon() { return fi # Start the daemon, wait for the socket to appear. - rm -f $NIX_DAEMON_SOCKET_PATH + rm -f "$NIX_DAEMON_SOCKET_PATH" PATH=$DAEMON_PATH nix --extra-experimental-features 'nix-command' daemon & _NIX_TEST_DAEMON_PID=$! export _NIX_TEST_DAEMON_PID @@ -151,14 +101,14 @@ killDaemon() { if [[ "${_NIX_TEST_DAEMON_PID-}" == '' ]]; then return fi - kill $_NIX_TEST_DAEMON_PID + kill "$_NIX_TEST_DAEMON_PID" for i in {0..100}; do - kill -0 $_NIX_TEST_DAEMON_PID 2> /dev/null || break + kill -0 "$_NIX_TEST_DAEMON_PID" 2> /dev/null || break sleep 0.1 done - kill -9 $_NIX_TEST_DAEMON_PID 2> /dev/null || true - wait $_NIX_TEST_DAEMON_PID || true - rm -f $NIX_DAEMON_SOCKET_PATH + kill -9 "$_NIX_TEST_DAEMON_PID" 2> /dev/null || true + wait "$_NIX_TEST_DAEMON_PID" || true + rm -f "$NIX_DAEMON_SOCKET_PATH" # Indicate daemon is stopped unset _NIX_TEST_DAEMON_PID # Restore old nix remote @@ -177,14 +127,11 @@ restartDaemon() { startDaemon } -if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then - _canUseSandbox=1 -fi - isDaemonNewer () { [[ -n "${NIX_DAEMON_PACKAGE:-}" ]] || return 0 local requiredVersion="$1" - local daemonVersion=$($NIX_DAEMON_PACKAGE/bin/nix daemon --version | cut -d' ' -f3) + local daemonVersion + daemonVersion=$("$NIX_DAEMON_PACKAGE/bin/nix" daemon --version | cut -d' ' -f3) [[ $(nix eval --expr "builtins.compareVersions ''$daemonVersion'' ''$requiredVersion''") -ge 0 ]] } @@ -237,7 +184,7 @@ expect() { shift "$@" && res=0 || res="$?" # also match "negative" codes, which wrap around to >127 - if [[ $res -ne $expected && $res -ne $[256 + expected] ]]; then + if [[ $res -ne $expected && $res -ne $((256 + expected)) ]]; then echo "Expected exit code '$expected' but got '$res' from command ${*@Q}" >&2 return 1 fi @@ -252,7 +199,7 @@ expectStderr() { shift "$@" 2>&1 && res=0 || res="$?" # also match "negative" codes, which wrap around to >127 - if [[ $res -ne $expected && $res -ne $[256 + expected] ]]; then + if [[ $res -ne $expected && $res -ne $((256 + expected)) ]]; then echo "Expected exit code '$expected' but got '$res' from command ${*@Q}" >&2 return 1 fi @@ -267,7 +214,7 @@ expectStderr() { # error: This error is expected # EOF assertStderr() { - diff -u /dev/stdin <($@ 2>/dev/null 2>&1) + diff -u /dev/stdin <("$@" 2>/dev/null 2>&1) } needLocalStore() { @@ -283,11 +230,9 @@ buggyNeedLocalStore() { enableFeatures() { local features="$1" - sed -i 's/experimental-features .*/& '"$features"'/' "$test_nix_conf_dir"/nix.conf + sed -i 's/experimental-features .*/& '"$features"'/' "${test_nix_conf?}" } -set -x - onError() { set +x echo "$0: test failed at:" >&2 @@ -311,15 +256,15 @@ callerPrefix() { local i file line fn savedFn # Use `caller` for i in $(seq 0 100); do - caller $i > /dev/null || { + caller "$i" > /dev/null || { if [[ -n "${file:-}" ]]; then echo "$file:$line: ${savedFn+in call to $savedFn: }" fi break } - line="$(caller $i | cut -d' ' -f1)" - fn="$(caller $i | cut -d' ' -f2)" - file="$(caller $i | cut -d' ' -f3)" + line="$(caller "$i" | cut -d' ' -f1)" + fn="$(caller "$i" | cut -d' ' -f2)" + file="$(caller "$i" | cut -d' ' -f3)" if [[ $file != "${BASH_SOURCE[0]}" ]]; then echo "$file:$line: ${savedFn+in call to $savedFn: }" return @@ -342,7 +287,7 @@ checkGrepArgs() { for arg in "$@"; do if [[ "$arg" != "${arg//$'\n'/_}" ]]; then echo "$(callerPrefix)newline not allowed in arguments; grep would try each line individually as if connected by an OR operator" >&2 - return -101 + return 155 # = -101 mod 256 fi done } @@ -400,4 +345,4 @@ count() { trap onError ERR -fi # COMMON_VARS_AND_FUNCTIONS_SH_SOURCED +fi # COMMON_FUNCTIONS_SH_SOURCED diff --git a/tests/functional/common/init.sh b/tests/functional/common/init.sh index d33ad5d57..d849c0734 100755 --- a/tests/functional/common/init.sh +++ b/tests/functional/common/init.sh @@ -7,10 +7,10 @@ if isTestOnNixOS; then mkdir -p "$test_nix_conf_dir" "$TEST_HOME" - export NIX_USER_CONF_FILES="$test_nix_conf_dir/nix.conf" + export NIX_USER_CONF_FILES="$test_nix_conf" mkdir -p "$test_nix_conf_dir" "$TEST_HOME" ! test -e "$test_nix_conf" - cat > "$test_nix_conf_dir/nix.conf" < "$test_nix_conf" < /dev/null); then + # Maybe the build directory is symlinked. + export NIX_IGNORE_SYMLINK_STORE=1 + NIX_STORE_DIR=$TEST_ROOT/store + fi + export NIX_LOCALSTATE_DIR=$TEST_ROOT/var + export NIX_LOG_DIR=$TEST_ROOT/var/log/nix + export NIX_STATE_DIR=$TEST_ROOT/var/nix + export NIX_CONF_DIR=$test_nix_conf_dir + export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/dSocket + unset NIX_USER_CONF_FILES + export _NIX_TEST_SHARED=$TEST_ROOT/shared + if [[ -n $NIX_STORE ]]; then + export _NIX_TEST_NO_SANDBOX=1 + fi + export _NIX_IN_TEST=$TEST_ROOT/shared + export _NIX_TEST_NO_LSOF=1 + export NIX_REMOTE=${NIX_REMOTE_-} + +fi # ! isTestOnNixOS + +unset NIX_PATH +export HOME=$TEST_HOME +unset XDG_STATE_HOME +unset XDG_DATA_HOME +unset XDG_CONFIG_HOME +unset XDG_CONFIG_DIRS +unset XDG_CACHE_HOME + +export IMPURE_VAR1=foo +export IMPURE_VAR2=bar + +# Used in other files +# shellcheck disable=SC2034 +cacheDir=$TEST_ROOT/binary-cache + +if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then + _canUseSandbox=1 +fi + +fi # COMMON_VARS_SH_SOURCED diff --git a/tests/functional/derivation-advanced-attributes.sh b/tests/functional/derivation-advanced-attributes.sh index 6c0c76b4c..271f17dc6 100755 --- a/tests/functional/derivation-advanced-attributes.sh +++ b/tests/functional/derivation-advanced-attributes.sh @@ -3,7 +3,7 @@ source common/test-root.sh source common/paths.sh -set -o pipefail +set -eu -o pipefail source characterisation/framework.sh diff --git a/tests/functional/dyn-drv/meson.build b/tests/functional/dyn-drv/meson.build new file mode 100644 index 000000000..3c671d013 --- /dev/null +++ b/tests/functional/dyn-drv/meson.build @@ -0,0 +1,19 @@ +configure_file( + input : 'config.nix.in', + output : 'config.nix', + configuration : test_confdata, +) + +suites += { + 'name': 'dyn-drv', + 'deps': [], + 'tests': [ + 'text-hashed-output.sh', + 'recursive-mod-json.sh', + 'build-built-drv.sh', + 'eval-outputOf.sh', + 'dep-built-drv.sh', + 'old-daemon-error-hack.sh', + ], + 'workdir': meson.current_build_dir(), +} diff --git a/tests/functional/flakes/meson.build b/tests/functional/flakes/meson.build new file mode 100644 index 000000000..8c1afd6ff --- /dev/null +++ b/tests/functional/flakes/meson.build @@ -0,0 +1,28 @@ +suites += { + 'name': 'flakes', + 'deps': [], + 'tests': [ + 'flakes.sh', + 'develop.sh', + 'edit.sh', + 'run.sh', + 'mercurial.sh', + 'circular.sh', + 'init.sh', + 'inputs.sh', + 'follow-paths.sh', + 'bundle.sh', + 'check.sh', + 'unlocked-override.sh', + 'absolute-paths.sh', + 'absolute-attr-paths.sh', + 'build-paths.sh', + 'flake-in-submodule.sh', + 'prefetch.sh', + 'eval-cache.sh', + 'search-root.sh', + 'config.sh', + 'show.sh', + ], + 'workdir': meson.current_build_dir(), +} diff --git a/tests/functional/git-hashing/meson.build b/tests/functional/git-hashing/meson.build new file mode 100644 index 000000000..7486bfb8f --- /dev/null +++ b/tests/functional/git-hashing/meson.build @@ -0,0 +1,8 @@ +suites += { + 'name': 'git-hashing', + 'deps': [], + 'tests': [ + 'simple.sh', + ], + 'workdir': meson.current_build_dir(), +} diff --git a/tests/functional/local-overlay-store/common.sh b/tests/functional/local-overlay-store/common.sh index b171f91f4..27338ea23 100644 --- a/tests/functional/local-overlay-store/common.sh +++ b/tests/functional/local-overlay-store/common.sh @@ -1,4 +1,5 @@ -source ../common/vars-and-functions.sh +source ../common/vars.sh +source ../common/functions.sh TODO_NixOS diff --git a/tests/functional/local-overlay-store/meson.build b/tests/functional/local-overlay-store/meson.build new file mode 100644 index 000000000..6ff5d3169 --- /dev/null +++ b/tests/functional/local-overlay-store/meson.build @@ -0,0 +1,18 @@ +suites += { + 'name': 'local-overlay-store', + 'deps': [], + 'tests': [ + 'check-post-init.sh', + 'redundant-add.sh', + 'build.sh', + 'bad-uris.sh', + 'add-lower.sh', + 'delete-refs.sh', + 'delete-duplicate.sh', + 'gc.sh', + 'verify.sh', + 'optimise.sh', + 'stale-file-handle.sh', + ], + 'workdir': meson.current_build_dir(), +} diff --git a/tests/functional/meson.build b/tests/functional/meson.build new file mode 100644 index 000000000..ebecdd9e8 --- /dev/null +++ b/tests/functional/meson.build @@ -0,0 +1,266 @@ +project('nix-functional-tests', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.3', + license : 'LGPL-2.1-or-later', +) + +fs = import('fs') + +# Need to combine source and build trees +run_command( + 'rsync', + '-a', + '--copy-unsafe-links', + meson.current_source_dir() / '', + meson.current_build_dir() / '', +) +# This current-source-escaping relative is no good because we don't know +# where the build directory will be, therefore we fix it up. Once the +# Make build system is gone, we should think about doing this better. +scripts_dir = fs.relative_to( + meson.current_source_dir() / '..' / '..' / 'scripts', + meson.current_build_dir(), +) +run_command( + 'sed', + '-i', meson.current_build_dir() / 'bash-profile.sh', + '-e', 's^../../scripts^@0@^'.format(scripts_dir), +) + +nix = find_program('nix') +bash = find_program('bash', native : true) +busybox = find_program('busybox', native : true, required : false) +coreutils = find_program('coreutils', native : true) +dot = find_program('dot', native : true, required : false) + +nix_bin_dir = fs.parent(nix.full_path()) + +test_confdata = { + 'bindir': nix_bin_dir, + 'coreutils': fs.parent(coreutils.full_path()), + 'dot': dot.found() ? dot.full_path() : '', + 'bash': bash.full_path(), + 'sandbox_shell': busybox.found() ? busybox.full_path() : '', + 'PACKAGE_VERSION': meson.project_version(), + 'system': host_machine.cpu_family() + '-' + host_machine.system(), +} + +# Just configures `common/vars-and-functions.sh.in`. +# Done as a subdir() so Meson places it under `common` in the build directory as well. +subdir('common') + +config_nix_in = configure_file( + input : 'config.nix.in', + output : 'config.nix', + configuration : test_confdata, +) + +suites = [ + { + 'name' : 'main', + 'deps': [], + 'tests': [ + 'test-infra.sh', + 'gc.sh', + 'nix-collect-garbage-d.sh', + 'remote-store.sh', + 'legacy-ssh-store.sh', + 'lang.sh', + 'lang-gc.sh', + 'characterisation-test-infra.sh', + 'experimental-features.sh', + 'fetchMercurial.sh', + 'gc-auto.sh', + 'user-envs.sh', + 'user-envs-migration.sh', + 'binary-cache.sh', + 'multiple-outputs.sh', + 'nix-build.sh', + 'gc-concurrent.sh', + 'repair.sh', + 'fixed.sh', + 'export-graph.sh', + 'timeout.sh', + 'fetchGitRefs.sh', + 'gc-runtime.sh', + 'tarball.sh', + 'fetchGit.sh', + 'fetchurl.sh', + 'fetchPath.sh', + 'fetchTree-file.sh', + 'simple.sh', + 'referrers.sh', + 'optimise-store.sh', + 'substitute-with-invalid-ca.sh', + 'signing.sh', + 'hash-convert.sh', + 'hash-path.sh', + 'gc-non-blocking.sh', + 'check.sh', + 'nix-shell.sh', + 'check-refs.sh', + 'build-remote-input-addressed.sh', + 'secure-drv-outputs.sh', + 'restricted.sh', + 'fetchGitSubmodules.sh', + 'fetchGitVerification.sh', + 'readfile-context.sh', + 'nix-channel.sh', + 'recursive.sh', + 'dependencies.sh', + 'check-reqs.sh', + 'build-remote-content-addressed-fixed.sh', + 'build-remote-content-addressed-floating.sh', + 'build-remote-trustless-should-pass-0.sh', + 'build-remote-trustless-should-pass-1.sh', + 'build-remote-trustless-should-pass-2.sh', + 'build-remote-trustless-should-pass-3.sh', + 'build-remote-trustless-should-fail-0.sh', + 'build-remote-with-mounted-ssh-ng.sh', + 'nar-access.sh', + 'impure-eval.sh', + 'pure-eval.sh', + 'eval.sh', + 'repl.sh', + 'binary-cache-build-remote.sh', + 'search.sh', + 'logging.sh', + 'export.sh', + 'config.sh', + 'add.sh', + 'chroot-store.sh', + 'filter-source.sh', + 'misc.sh', + 'dump-db.sh', + 'linux-sandbox.sh', + 'supplementary-groups.sh', + 'build-dry.sh', + 'structured-attrs.sh', + 'shell.sh', + 'brotli.sh', + 'zstd.sh', + 'compression-levels.sh', + 'nix-copy-ssh.sh', + 'nix-copy-ssh-ng.sh', + 'post-hook.sh', + 'function-trace.sh', + 'fmt.sh', + 'eval-store.sh', + 'why-depends.sh', + 'derivation-json.sh', + 'derivation-advanced-attributes.sh', + 'import-derivation.sh', + 'nix_path.sh', + 'case-hack.sh', + 'placeholders.sh', + 'ssh-relay.sh', + 'build.sh', + 'build-delete.sh', + 'output-normalization.sh', + 'selfref-gc.sh', + 'db-migration.sh', + 'bash-profile.sh', + 'pass-as-file.sh', + 'nix-profile.sh', + 'suggestions.sh', + 'store-info.sh', + 'fetchClosure.sh', + 'completions.sh', + 'impure-derivations.sh', + 'path-from-hash-part.sh', + 'path-info.sh', + 'toString-path.sh', + 'read-only-store.sh', + 'nested-sandboxing.sh', + 'impure-env.sh', + 'debugger.sh', + 'extra-sandbox-profile.sh', + 'help.sh', + ], + 'workdir': meson.current_build_dir(), + }, +] + +nix_store = dependency('nix-store', required : false) +if nix_store.found() + subdir('test-libstoreconsumer') + suites += { + 'name': 'libstoreconsumer', + 'deps': [ + libstoreconsumer_tester, + ], + 'tests': [ + 'test-libstoreconsumer.sh', + ], + 'workdir': meson.current_build_dir(), + } + +endif + +# Plugin tests require shared libraries support. +nix_expr = dependency('nix-expr', required : false) +if nix_expr.found() and get_option('default_library') != 'static' + subdir('plugins') + suites += { + 'name': 'plugins', + 'deps': [ + libplugintest, + ], + 'tests': [ + 'plugins.sh', + ], + 'workdir': meson.current_build_dir(), + } +endif + +subdir('ca') +subdir('dyn-drv') +subdir('flakes') +subdir('git-hashing') +subdir('local-overlay-store') + +foreach suite : suites + foreach script : suite['tests'] + workdir = suite['workdir'] + prefix = fs.relative_to(workdir, meson.project_build_root()) + + script = script + # Turns, e.g., `tests/functional/flakes/show.sh` into a Meson test target called + # `functional-flakes-show`. + name = fs.replace_suffix(prefix / script, '') + + test( + name, + bash, + args: [ + '-x', + '-e', + '-u', + '-o', 'pipefail', + script, + ], + suite : suite['name'], + env : { + 'TEST_NAME': name, + 'NIX_REMOTE': '', + 'PS4': '+(${BASH_SOURCE[0]-$0}:$LINENO) ', + }, + # some tests take 15+ seconds even on an otherwise idle machine, on a loaded machine + # this can easily drive them to failure. give them more time than default of 30sec + timeout : 300, + # Used for target dependency/ordering tracking, not adding compiler flags or anything. + depends : suite['deps'], + workdir : workdir, + # Won't pass until man pages are generated + should_fail : suite['name'] == 'main' and script == 'help.sh' + ) + endforeach +endforeach diff --git a/tests/functional/nested-sandboxing/command.sh b/tests/functional/nested-sandboxing/command.sh index 69366486c..e9c40a5d9 100644 --- a/tests/functional/nested-sandboxing/command.sh +++ b/tests/functional/nested-sandboxing/command.sh @@ -1,3 +1,5 @@ +set -eu -o pipefail + export NIX_BIN_DIR=$(dirname $(type -p nix)) # TODO Get Nix and its closure more flexibly export EXTRA_SANDBOX="/nix/store $(dirname $NIX_BIN_DIR)" diff --git a/tests/functional/nested-sandboxing/runner.nix b/tests/functional/nested-sandboxing/runner.nix index 9a5822c88..1e79d5065 100644 --- a/tests/functional/nested-sandboxing/runner.nix +++ b/tests/functional/nested-sandboxing/runner.nix @@ -6,7 +6,10 @@ mkDerivation { name = "nested-sandboxing"; busybox = builtins.getEnv "busybox"; EXTRA_SANDBOX = builtins.getEnv "EXTRA_SANDBOX"; - buildCommand = if altitude == 0 then '' + buildCommand = '' + set -x + set -eu -o pipefail + '' + (if altitude == 0 then '' echo Deep enough! > $out '' else '' cp -r ${../common} ./common @@ -20,5 +23,5 @@ mkDerivation { source ./nested-sandboxing/command.sh runNixBuild ${storeFun} ${toString altitude} >> $out - ''; + ''); } diff --git a/tests/functional/package.nix b/tests/functional/package.nix new file mode 100644 index 000000000..205b03614 --- /dev/null +++ b/tests/functional/package.nix @@ -0,0 +1,117 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config +, rsync + +, jq +, git +, mercurial +, util-linux + +, nix-store +, nix-expr +, nix-ng + +, rapidcheck +, gtest +, runCommand + +, busybox-sandbox-shell ? null + +# Configuration Options + +, version + +# For running the functional tests against a different pre-built Nix. +, test-daemon ? null +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-functional-tests"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../scripts/nix-profile.sh.in + ../../.version + ../../tests/functional + ./. + ]; + + # Hack for sake of the dev shell + passthru.baseNativeBuildInputs = [ + meson + ninja + pkg-config + rsync + + jq + git + mercurial + ] ++ lib.optionals stdenv.hostPlatform.isLinux [ + # For various sandboxing tests that needs a statically-linked shell, + # etc. + busybox-sandbox-shell + # For Overlay FS tests need `mount`, `umount`, and `unshare`. + # TODO use `unixtools` to be precise over which executables instead? + util-linux + ]; + + nativeBuildInputs = finalAttrs.passthru.baseNativeBuildInputs ++ [ + nix-ng + ]; + + buildInputs = [ + nix-store + nix-expr + ]; + + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../../.version + '' + # TEMP hack for Meson before make is gone, where + # `src/nix-functional-tests` is during the transition a symlink and + # not the actual directory directory. + + '' + cd $(readlink -e $PWD) + echo $PWD | grep tests/functional + ''; + + mesonCheckFlags = [ + "--print-errorlogs" + ]; + + preCheck = + # See https://github.com/NixOS/nix/issues/2523 + # Occurs often in tests since https://github.com/NixOS/nix/pull/9900 + lib.optionalString stdenv.hostPlatform.isDarwin '' + export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES + ''; + + doCheck = true; + + installPhase = '' + touch $out + ''; + + meta = { + platforms = lib.platforms.unix; + }; + +} // lib.optionalAttrs (test-daemon != null) { + NIX_DAEMON_PACKAGE = test-daemon; +}) diff --git a/tests/functional/plugins.sh b/tests/functional/plugins.sh index ab4876df9..fc2d1907c 100755 --- a/tests/functional/plugins.sh +++ b/tests/functional/plugins.sh @@ -2,10 +2,11 @@ source common.sh -if [[ $BUILD_SHARED_LIBS != 1 ]]; then - skipTest "Plugins are not supported" -fi +for ext in so dylib; do + plugin="$PWD/plugins/libplugintest.$ext" + [[ -f "$plugin" ]] && break +done -res=$(nix --option setting-set true --option plugin-files $PWD/plugins/libplugintest* eval --expr builtins.anotherNull) +res=$(nix --option setting-set true --option plugin-files "$plugin" eval --expr builtins.anotherNull) [ "$res"x = "nullx" ] diff --git a/tests/functional/plugins/meson.build b/tests/functional/plugins/meson.build new file mode 100644 index 000000000..3d6b2f0e1 --- /dev/null +++ b/tests/functional/plugins/meson.build @@ -0,0 +1,16 @@ +libplugintest = shared_module( + 'plugintest', + 'plugintest.cc', + cpp_args : [ + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + # '-include', 'config-fetchers.hh', + '-include', 'config-expr.hh', + ], + dependencies : [ + dependency('nix-expr'), + ], + build_by_default : false, +) diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh index 591367e9f..e5fe9c136 100755 --- a/tests/functional/restricted.sh +++ b/tests/functional/restricted.sh @@ -16,9 +16,6 @@ nix-instantiate --restrict-eval ./simple.nix -I src1=simple.nix -I src2=config.n (! nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix') nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I src=../.. -(! nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel') -nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel' -I src=../../src - expectStderr 1 nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile ' | grepQuiet "forbidden in restricted mode" nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile ' -I src=. diff --git a/tests/functional/test-libstoreconsumer/meson.build b/tests/functional/test-libstoreconsumer/meson.build new file mode 100644 index 000000000..7076127f7 --- /dev/null +++ b/tests/functional/test-libstoreconsumer/meson.build @@ -0,0 +1,14 @@ +libstoreconsumer_tester = executable( + 'test-libstoreconsumer', + 'main.cc', + cpp_args : [ + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + ], + dependencies : [ + dependency('nix-store'), + ], + build_by_default : false, +) From 6f3045c2a225dca7b1ed8a9c9dc27ab50f575900 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 14 Aug 2024 15:40:32 -0400 Subject: [PATCH 136/142] Remove unit tests from old build system Now that we can run all tests with Meson, we want developers making code changes to use it. (Only the manual needs to be built with the build system, and that will change shortly.) This reverts commit b0bc2a97bfe007fbc32f584ed9de5e7cb75a521c. --- Makefile | 19 ------------ Makefile.config.in | 1 - configure.ac | 22 -------------- package.nix | 33 +------------------- tests/unit/libexpr-support/local.mk | 23 -------------- tests/unit/libexpr/local.mk | 45 ---------------------------- tests/unit/libfetchers/local.mk | 37 ----------------------- tests/unit/libflake/local.mk | 43 -------------------------- tests/unit/libstore-support/local.mk | 21 ------------- tests/unit/libstore/local.mk | 38 ----------------------- tests/unit/libutil-support/local.mk | 19 ------------ tests/unit/libutil/local.mk | 37 ----------------------- 12 files changed, 1 insertion(+), 337 deletions(-) delete mode 100644 tests/unit/libexpr-support/local.mk delete mode 100644 tests/unit/libexpr/local.mk delete mode 100644 tests/unit/libfetchers/local.mk delete mode 100644 tests/unit/libflake/local.mk delete mode 100644 tests/unit/libstore-support/local.mk delete mode 100644 tests/unit/libstore/local.mk delete mode 100644 tests/unit/libutil-support/local.mk delete mode 100644 tests/unit/libutil/local.mk diff --git a/Makefile b/Makefile index dbf510a3e..b51ae6cc7 100644 --- a/Makefile +++ b/Makefile @@ -38,18 +38,6 @@ makefiles += \ endif endif -ifeq ($(ENABLE_UNIT_TESTS), yes) -makefiles += \ - tests/unit/libutil/local.mk \ - tests/unit/libutil-support/local.mk \ - tests/unit/libstore/local.mk \ - tests/unit/libstore-support/local.mk \ - tests/unit/libfetchers/local.mk \ - tests/unit/libexpr/local.mk \ - tests/unit/libexpr-support/local.mk \ - tests/unit/libflake/local.mk -endif - ifeq ($(ENABLE_FUNCTIONAL_TESTS), yes) ifdef HOST_UNIX makefiles += \ @@ -104,13 +92,6 @@ include mk/lib.mk # These must be defined after `mk/lib.mk`. Otherwise the first rule # incorrectly becomes the default target. -ifneq ($(ENABLE_UNIT_TESTS), yes) -.PHONY: check -check: - @echo "Unit tests are disabled. Configure without '--disable-unit-tests', or avoid calling 'make check'." - @exit 1 -endif - ifneq ($(ENABLE_FUNCTIONAL_TESTS), yes) .PHONY: installcheck installcheck: diff --git a/Makefile.config.in b/Makefile.config.in index 3100d2073..e131484f6 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -12,7 +12,6 @@ ENABLE_BUILD = @ENABLE_BUILD@ ENABLE_DOC_GEN = @ENABLE_DOC_GEN@ ENABLE_FUNCTIONAL_TESTS = @ENABLE_FUNCTIONAL_TESTS@ ENABLE_S3 = @ENABLE_S3@ -ENABLE_UNIT_TESTS = @ENABLE_UNIT_TESTS@ GTEST_LIBS = @GTEST_LIBS@ HAVE_LIBCPUID = @HAVE_LIBCPUID@ HAVE_SECCOMP = @HAVE_SECCOMP@ diff --git a/configure.ac b/configure.ac index 5c22ed176..18d718c07 100644 --- a/configure.ac +++ b/configure.ac @@ -141,18 +141,6 @@ AC_ARG_ENABLE(build, AS_HELP_STRING([--disable-build],[Do not build nix]), ENABLE_BUILD=$enableval, ENABLE_BUILD=yes) AC_SUBST(ENABLE_BUILD) -# Building without unit tests is useful for bootstrapping with a smaller footprint -# or running the tests in a separate derivation. Otherwise, we do compile and -# run them. - -AC_ARG_ENABLE(unit-tests, AS_HELP_STRING([--disable-unit-tests],[Do not build the tests]), - ENABLE_UNIT_TESTS=$enableval, ENABLE_UNIT_TESTS=$ENABLE_BUILD) -AC_SUBST(ENABLE_UNIT_TESTS) - -AS_IF( - [test "$ENABLE_BUILD" == "no" && test "$ENABLE_UNIT_TESTS" == "yes"], - [AC_MSG_ERROR([Cannot enable unit tests when building overall is disabled. Please do not pass '--enable-unit-tests' or do not pass '--disable-build'.])]) - AC_ARG_ENABLE(functional-tests, AS_HELP_STRING([--disable-functional-tests],[Do not build the tests]), ENABLE_FUNCTIONAL_TESTS=$enableval, ENABLE_FUNCTIONAL_TESTS=yes) AC_SUBST(ENABLE_FUNCTIONAL_TESTS) @@ -358,16 +346,6 @@ if test "$gc" = yes; then CFLAGS="$old_CFLAGS" fi -AS_IF([test "$ENABLE_UNIT_TESTS" == "yes"],[ - -# Look for gtest. -PKG_CHECK_MODULES([GTEST], [gtest_main gmock_main]) - -# Look for rapidcheck. -PKG_CHECK_MODULES([RAPIDCHECK], [rapidcheck rapidcheck_gtest]) - -]) - # Look for nlohmann/json. PKG_CHECK_MODULES([NLOHMANN_JSON], [nlohmann_json >= 3.9]) diff --git a/package.nix b/package.nix index d41748b7c..c0d04179d 100644 --- a/package.nix +++ b/package.nix @@ -53,10 +53,6 @@ # Whether to build Nix. Useful to skip for tasks like testing existing pre-built versions of Nix , doBuild ? true -# Run the unit tests as part of the build. See `installUnitTests` for an -# alternative to this. -, doCheck ? __forDefaults.canRunInstalled - # Run the functional tests as part of the build. , doInstallCheck ? test-client != null || __forDefaults.canRunInstalled @@ -89,11 +85,6 @@ # - readline , readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline" -# Whether to install unit tests. This is useful when cross compiling -# since we cannot run them natively during the build, but can do so -# later. -, installUnitTests ? doBuild && !__forDefaults.canExecuteHost - # For running the functional tests against a pre-built Nix. Probably # want to use in conjunction with `doBuild = false;`. , test-daemon ? null @@ -117,7 +108,7 @@ let # things which should instead be gotten via `finalAttrs` in order to # work with overriding. attrs = { - inherit doBuild doCheck doInstallCheck; + inherit doBuild doInstallCheck; }; mkDerivation = @@ -133,16 +124,11 @@ in mkDerivation (finalAttrs: let inherit (finalAttrs) - doCheck doInstallCheck ; doBuild = !finalAttrs.dontBuild; - # Either running the unit tests during the build, or installing them - # to be run later, requiresthe unit tests to be built. - buildUnitTests = doCheck || installUnitTests; - in { inherit pname version; @@ -176,8 +162,6 @@ in { ./scripts/local.mk ] ++ lib.optionals enableManual [ ./doc/manual - ] ++ lib.optionals buildUnitTests [ - ./tests/unit ] ++ lib.optionals doInstallCheck [ ./tests/functional ])); @@ -190,8 +174,6 @@ in { # If we are doing just build or just docs, the one thing will use # "out". We only need additional outputs if we are doing both. ++ lib.optional (doBuild && enableManual) "doc" - ++ lib.optional installUnitTests "check" - ++ lib.optional doCheck "testresults" ; nativeBuildInputs = [ @@ -230,9 +212,6 @@ in { ({ inherit readline editline; }.${readlineFlavor}) ] ++ lib.optionals enableMarkdown [ lowdown - ] ++ lib.optionals buildUnitTests [ - gtest - rapidcheck ] ++ lib.optional stdenv.isLinux libseccomp ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid # There have been issues building these dependencies @@ -247,22 +226,16 @@ in { ); dontBuild = !attrs.doBuild; - doCheck = attrs.doCheck; configureFlags = [ (lib.enableFeature doBuild "build") - (lib.enableFeature buildUnitTests "unit-tests") (lib.enableFeature doInstallCheck "functional-tests") (lib.enableFeature enableManual "doc-gen") (lib.enableFeature enableGC "gc") (lib.enableFeature enableMarkdown "markdown") - (lib.enableFeature installUnitTests "install-unit-tests") (lib.withFeatureAs true "readline-flavor" readlineFlavor) ] ++ lib.optionals (!forDevShell) [ "--sysconfdir=/etc" - ] ++ lib.optionals installUnitTests [ - "--with-check-bin-dir=${builtins.placeholder "check"}/bin" - "--with-check-lib-dir=${builtins.placeholder "check"}/lib" ] ++ lib.optionals (doBuild) [ "--with-boost=${boost}/lib" ] ++ lib.optionals (doBuild && stdenv.isLinux) [ @@ -343,10 +316,6 @@ in { platforms = lib.platforms.unix ++ lib.platforms.windows; mainProgram = "nix"; broken = !(lib.all (a: a) [ - # We cannot run or install unit tests if we don't build them or - # Nix proper (which they depend on). - (installUnitTests -> doBuild) - (doCheck -> doBuild) # The build process for the manual currently requires extracting # data from the Nix executable we are trying to document. (enableManual -> doBuild) diff --git a/tests/unit/libexpr-support/local.mk b/tests/unit/libexpr-support/local.mk deleted file mode 100644 index 0501de33c..000000000 --- a/tests/unit/libexpr-support/local.mk +++ /dev/null @@ -1,23 +0,0 @@ -libraries += libexpr-test-support - -libexpr-test-support_NAME = libnixexpr-test-support - -libexpr-test-support_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libexpr-test-support_INSTALL_DIR := $(checklibdir) -else - libexpr-test-support_INSTALL_DIR := -endif - -libexpr-test-support_SOURCES := \ - $(wildcard $(d)/tests/*.cc) \ - $(wildcard $(d)/tests/value/*.cc) - -libexpr-test-support_CXXFLAGS += $(libexpr-tests_EXTRA_INCLUDES) - -libexpr-test-support_LIBS = \ - libstore-test-support libutil-test-support \ - libexpr libstore libutil - -libexpr-test-support_LDFLAGS := $(THREAD_LDFLAGS) -lrapidcheck diff --git a/tests/unit/libexpr/local.mk b/tests/unit/libexpr/local.mk deleted file mode 100644 index 1617e2823..000000000 --- a/tests/unit/libexpr/local.mk +++ /dev/null @@ -1,45 +0,0 @@ -check: libexpr-tests_RUN - -programs += libexpr-tests - -libexpr-tests_NAME := libnixexpr-tests - -libexpr-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libexpr-tests.xml - -libexpr-tests_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libexpr-tests_INSTALL_DIR := $(checkbindir) -else - libexpr-tests_INSTALL_DIR := -endif - -libexpr-tests_SOURCES := \ - $(wildcard $(d)/*.cc) \ - $(wildcard $(d)/value/*.cc) \ - $(wildcard $(d)/flake/*.cc) - -libexpr-tests_EXTRA_INCLUDES = \ - -I tests/unit/libexpr-support \ - -I tests/unit/libstore-support \ - -I tests/unit/libutil-support \ - $(INCLUDE_libexpr) \ - $(INCLUDE_libexprc) \ - $(INCLUDE_libfetchers) \ - $(INCLUDE_libstore) \ - $(INCLUDE_libstorec) \ - $(INCLUDE_libutil) \ - $(INCLUDE_libutilc) - -libexpr-tests_CXXFLAGS += $(libexpr-tests_EXTRA_INCLUDES) - -libexpr-tests_LIBS = \ - libexpr-test-support libstore-test-support libutil-test-support \ - libexpr libexprc libfetchers libstore libstorec libutil libutilc - -libexpr-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock - -ifdef HOST_WINDOWS - # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space - libexpr-tests_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024))) -endif diff --git a/tests/unit/libfetchers/local.mk b/tests/unit/libfetchers/local.mk deleted file mode 100644 index 30aa142a5..000000000 --- a/tests/unit/libfetchers/local.mk +++ /dev/null @@ -1,37 +0,0 @@ -check: libfetchers-tests_RUN - -programs += libfetchers-tests - -libfetchers-tests_NAME = libnixfetchers-tests - -libfetchers-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libfetchers-tests.xml - -libfetchers-tests_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libfetchers-tests_INSTALL_DIR := $(checkbindir) -else - libfetchers-tests_INSTALL_DIR := -endif - -libfetchers-tests_SOURCES := $(wildcard $(d)/*.cc) - -libfetchers-tests_EXTRA_INCLUDES = \ - -I tests/unit/libstore-support \ - -I tests/unit/libutil-support \ - $(INCLUDE_libfetchers) \ - $(INCLUDE_libstore) \ - $(INCLUDE_libutil) - -libfetchers-tests_CXXFLAGS += $(libfetchers-tests_EXTRA_INCLUDES) - -libfetchers-tests_LIBS = \ - libstore-test-support libutil-test-support \ - libfetchers libstore libutil - -libfetchers-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) $(LIBGIT2_LIBS) - -ifdef HOST_WINDOWS - # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space - libfetchers-tests_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024))) -endif diff --git a/tests/unit/libflake/local.mk b/tests/unit/libflake/local.mk deleted file mode 100644 index 590bcf7c0..000000000 --- a/tests/unit/libflake/local.mk +++ /dev/null @@ -1,43 +0,0 @@ -check: libflake-tests_RUN - -programs += libflake-tests - -libflake-tests_NAME := libnixflake-tests - -libflake-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libflake-tests.xml - -libflake-tests_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libflake-tests_INSTALL_DIR := $(checkbindir) -else - libflake-tests_INSTALL_DIR := -endif - -libflake-tests_SOURCES := \ - $(wildcard $(d)/*.cc) \ - $(wildcard $(d)/value/*.cc) \ - $(wildcard $(d)/flake/*.cc) - -libflake-tests_EXTRA_INCLUDES = \ - -I tests/unit/libflake-support \ - -I tests/unit/libstore-support \ - -I tests/unit/libutil-support \ - $(INCLUDE_libflake) \ - $(INCLUDE_libexpr) \ - $(INCLUDE_libfetchers) \ - $(INCLUDE_libstore) \ - $(INCLUDE_libutil) \ - -libflake-tests_CXXFLAGS += $(libflake-tests_EXTRA_INCLUDES) - -libflake-tests_LIBS = \ - libexpr-test-support libstore-test-support libutil-test-support \ - libflake libexpr libfetchers libstore libutil - -libflake-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock - -ifdef HOST_WINDOWS - # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space - libflake-tests_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024))) -endif diff --git a/tests/unit/libstore-support/local.mk b/tests/unit/libstore-support/local.mk deleted file mode 100644 index 56dedd825..000000000 --- a/tests/unit/libstore-support/local.mk +++ /dev/null @@ -1,21 +0,0 @@ -libraries += libstore-test-support - -libstore-test-support_NAME = libnixstore-test-support - -libstore-test-support_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libstore-test-support_INSTALL_DIR := $(checklibdir) -else - libstore-test-support_INSTALL_DIR := -endif - -libstore-test-support_SOURCES := $(wildcard $(d)/tests/*.cc) - -libstore-test-support_CXXFLAGS += $(libstore-tests_EXTRA_INCLUDES) - -libstore-test-support_LIBS = \ - libutil-test-support \ - libstore libutil - -libstore-test-support_LDFLAGS := $(THREAD_LDFLAGS) -lrapidcheck diff --git a/tests/unit/libstore/local.mk b/tests/unit/libstore/local.mk deleted file mode 100644 index 8d3d6b0af..000000000 --- a/tests/unit/libstore/local.mk +++ /dev/null @@ -1,38 +0,0 @@ -check: libstore-tests_RUN - -programs += libstore-tests - -libstore-tests_NAME = libnixstore-tests - -libstore-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libstore-tests.xml - -libstore-tests_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libstore-tests_INSTALL_DIR := $(checkbindir) -else - libstore-tests_INSTALL_DIR := -endif - -libstore-tests_SOURCES := $(wildcard $(d)/*.cc) - -libstore-tests_EXTRA_INCLUDES = \ - -I tests/unit/libstore-support \ - -I tests/unit/libutil-support \ - $(INCLUDE_libstore) \ - $(INCLUDE_libstorec) \ - $(INCLUDE_libutil) \ - $(INCLUDE_libutilc) - -libstore-tests_CXXFLAGS += $(libstore-tests_EXTRA_INCLUDES) - -libstore-tests_LIBS = \ - libstore-test-support libutil-test-support \ - libstore libstorec libutil libutilc - -libstore-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) - -ifdef HOST_WINDOWS - # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space - libstore-tests_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024))) -endif diff --git a/tests/unit/libutil-support/local.mk b/tests/unit/libutil-support/local.mk deleted file mode 100644 index 5f7835c9f..000000000 --- a/tests/unit/libutil-support/local.mk +++ /dev/null @@ -1,19 +0,0 @@ -libraries += libutil-test-support - -libutil-test-support_NAME = libnixutil-test-support - -libutil-test-support_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libutil-test-support_INSTALL_DIR := $(checklibdir) -else - libutil-test-support_INSTALL_DIR := -endif - -libutil-test-support_SOURCES := $(wildcard $(d)/tests/*.cc) - -libutil-test-support_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES) - -libutil-test-support_LIBS = libutil - -libutil-test-support_LDFLAGS := $(THREAD_LDFLAGS) -lrapidcheck diff --git a/tests/unit/libutil/local.mk b/tests/unit/libutil/local.mk deleted file mode 100644 index 404f35cf1..000000000 --- a/tests/unit/libutil/local.mk +++ /dev/null @@ -1,37 +0,0 @@ -check: libutil-tests_RUN - -programs += libutil-tests - -libutil-tests_NAME = libnixutil-tests - -libutil-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libutil-tests.xml - -libutil-tests_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libutil-tests_INSTALL_DIR := $(checkbindir) -else - libutil-tests_INSTALL_DIR := -endif - -libutil-tests_SOURCES := $(wildcard $(d)/*.cc) - -libutil-tests_EXTRA_INCLUDES = \ - -I tests/unit/libutil-support \ - $(INCLUDE_libutil) \ - $(INCLUDE_libutilc) - -libutil-tests_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES) - -libutil-tests_LIBS = libutil-test-support libutil libutilc - -libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) - -ifdef HOST_WINDOWS - # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space - libutil-tests_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024))) -endif - -check: $(d)/data/git/check-data.sh.test - -$(eval $(call run-test,$(d)/data/git/check-data.sh)) From b41cc1a7555e59d5753bf4f8cbbfa9137107d2fe Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 14 Aug 2024 16:04:35 -0400 Subject: [PATCH 137/142] Make wrapper derivation This ensures just `nix build`-ing the flake doesn't forget to run all tests. One can still specifiy specific attributes to just build one thing. Co-authored-by: Robert Hensing --- .github/workflows/ci.yml | 14 ------ flake.nix | 51 ++++++++++++++------ packaging/components.nix | 6 ++- packaging/everything.nix | 93 ++++++++++++++++++++++++++++++++++++ packaging/hydra.nix | 5 +- tests/functional/package.nix | 6 +-- tests/nixos/quick-build.nix | 6 +-- 7 files changed, 142 insertions(+), 39 deletions(-) create mode 100644 packaging/everything.nix diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4eb9cf10d..9831d0e0e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -195,20 +195,6 @@ jobs: - uses: DeterminateSystems/magic-nix-cache-action@main - run: nix build -L .#hydraJobs.tests.githubFlakes .#hydraJobs.tests.tarballFlakes .#hydraJobs.tests.functional_user - meson_build: - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - uses: DeterminateSystems/nix-installer-action@main - - uses: DeterminateSystems/magic-nix-cache-action@main - # Only meson packages that don't have a tests.run derivation. - # Those that have it are already built and tested as part of nix flake check. - - run: nix build -L .#hydraJobs.build.{nix-cmd,nix-main}.$(nix-instantiate --eval --expr builtins.currentSystem | sed -e 's/"//g') - flake_regressions: needs: vm_tests runs-on: ubuntu-22.04 diff --git a/flake.nix b/flake.nix index 22ca54118..cded6c3a9 100644 --- a/flake.nix +++ b/flake.nix @@ -139,11 +139,6 @@ nix = final.nixComponents.nix; - nix_noTests = final.nix.override { - doInstallCheck = false; - doCheck = false; - }; - # See https://github.com/NixOS/nixpkgs/pull/214409 # Remove when fixed in this flake's nixpkgs pre-commit = @@ -222,7 +217,7 @@ # for which we don't apply the full build matrix such as cross or static. inherit (nixpkgsFor.${system}.native) changelog-d; - default = self.packages.${system}.nix; + default = self.packages.${system}.nix-ng; nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-internal-api-docs; nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-external-api-docs; } @@ -230,22 +225,48 @@ // flatMapAttrs { # Components we'll iterate over in the upcoming lambda "nix" = { }; - # Temporarily disabled because GitHub Actions OOM issues. Once - # the old build system is gone and we are back to one build - # system, we should reenable these. - #"nix-util" = { }; - #"nix-store" = { }; - #"nix-fetchers" = { }; + "nix-util" = { }; + "nix-util-c" = { }; + "nix-util-test-support" = { }; + "nix-util-tests" = { }; + + "nix-store" = { }; + "nix-store-c" = { }; + "nix-store-test-support" = { }; + "nix-store-tests" = { }; + + "nix-fetchers" = { }; + "nix-fetchers-tests" = { }; + + "nix-expr" = { }; + "nix-expr-c" = { }; + "nix-expr-test-support" = { }; + "nix-expr-tests" = { }; + + "nix-flake" = { }; + "nix-flake-tests" = { }; + + "nix-main" = { }; + "nix-main-c" = { }; + + "nix-cmd" = { }; + + "nix-cli" = { }; + + "nix-functional-tests" = { supportsCross = false; }; + + "nix-perl-bindings" = { supportsCross = false; }; + "nix-ng" = { }; } - (pkgName: {}: { + (pkgName: { supportsCross ? true }: { # These attributes go right into `packages.`. "${pkgName}" = nixpkgsFor.${system}.native.nixComponents.${pkgName}; "${pkgName}-static" = nixpkgsFor.${system}.static.nixComponents.${pkgName}; } - // flatMapAttrs (lib.genAttrs crossSystems (_: { })) (crossSystem: {}: { + // lib.optionalAttrs supportsCross (flatMapAttrs (lib.genAttrs crossSystems (_: { })) (crossSystem: {}: { # These attributes go right into `packages.`. "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}; - }) + })) // flatMapAttrs (lib.genAttrs stdenvs (_: { })) (stdenvName: {}: { # These attributes go right into `packages.`. "${pkgName}-${stdenvName}" = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nixComponents.${pkgName}; diff --git a/packaging/components.nix b/packaging/components.nix index f14613e8a..5fc3236cf 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -56,8 +56,7 @@ in nix-cmd = callPackage ../src/libcmd/package.nix { }; - # Will replace `nix` once the old build system is gone. - nix-ng = callPackage ../src/nix/package.nix { version = fineVersion; }; + nix-cli = callPackage ../src/nix/package.nix { version = fineVersion; }; nix-functional-tests = callPackage ../src/nix-functional-tests/package.nix { version = fineVersion; }; @@ -65,4 +64,7 @@ in nix-external-api-docs = callPackage ../src/external-api-docs/package.nix { version = fineVersion; }; nix-perl-bindings = callPackage ../src/perl/package.nix { }; + + # Will replace `nix` once the old build system is gone. + nix-ng = callPackage ../packaging/everything.nix { }; } diff --git a/packaging/everything.nix b/packaging/everything.nix new file mode 100644 index 000000000..8c8ce6611 --- /dev/null +++ b/packaging/everything.nix @@ -0,0 +1,93 @@ +{ + lib, + stdenv, + buildEnv, + + nix-util, + nix-util-c, + nix-util-test-support, + nix-util-tests, + + nix-store, + nix-store-c, + nix-store-test-support, + nix-store-tests, + + nix-fetchers, + nix-fetchers-tests, + + nix-expr, + nix-expr-c, + nix-expr-test-support, + nix-expr-tests, + + nix-flake, + nix-flake-tests, + + nix-main, + nix-main-c, + + nix-cmd, + + nix-cli, + + nix-functional-tests, + + nix-internal-api-docs, + nix-external-api-docs, + + nix-perl-bindings, +}: + +(buildEnv rec { + name = "nix-${nix-cli.version}"; + paths = [ + nix-util + nix-util-c + nix-util-test-support + nix-util-tests + + nix-store + nix-store-c + nix-store-test-support + nix-store-tests + + nix-fetchers + nix-fetchers-tests + + nix-expr + nix-expr-c + nix-expr-test-support + nix-expr-tests + + nix-flake + nix-flake-tests + + nix-main + nix-main-c + + nix-cmd + + nix-cli + + nix-internal-api-docs + nix-external-api-docs + + ] ++ lib.optionals (stdenv.buildPlatform.canExecute stdenv.hostPlatform) [ + nix-perl-bindings + ]; +}).overrideAttrs (_: { + doCheck = true; + doInstallCheck = true; + + checkInputs = [ + # Actually run the unit tests too + nix-util-tests.tests.run + nix-store-tests.tests.run + nix-expr-tests.tests.run + nix-flake-tests.tests.run + ]; + installCheckInputs = [ + nix-functional-tests + ]; +}) diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 46b4ff51d..65978835c 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -61,8 +61,9 @@ let "nix-main" "nix-main-c" "nix-cmd" - "nix-ng" + "nix-cli" "nix-functional-tests" + "nix-ng" ]; in { @@ -85,7 +86,7 @@ in self.packages.${system}.nix.override { enableGC = false; } ); - buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.nix_noTests); + buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.nix-cli); # Toggles some settings for better coverage. Windows needs these # library combinations, and Debian build Nix with GNU readline too. diff --git a/tests/functional/package.nix b/tests/functional/package.nix index 205b03614..277711123 100644 --- a/tests/functional/package.nix +++ b/tests/functional/package.nix @@ -15,7 +15,7 @@ , nix-store , nix-expr -, nix-ng +, nix-cli , rapidcheck , gtest @@ -67,7 +67,7 @@ mkMesonDerivation (finalAttrs: { ]; nativeBuildInputs = finalAttrs.passthru.baseNativeBuildInputs ++ [ - nix-ng + nix-cli ]; buildInputs = [ @@ -105,7 +105,7 @@ mkMesonDerivation (finalAttrs: { doCheck = true; installPhase = '' - touch $out + mkdir $out ''; meta = { diff --git a/tests/nixos/quick-build.nix b/tests/nixos/quick-build.nix index 37169b4e2..57e3b9cdb 100644 --- a/tests/nixos/quick-build.nix +++ b/tests/nixos/quick-build.nix @@ -27,13 +27,13 @@ in }; config = { - passthru.quickBuild = + passthru.quickBuild = let withQuickBuild = extendModules { modules = [{ quickBuild = true; }]; }; in withQuickBuild.config.test; defaults = { pkgs, ... }: { config = lib.mkIf test.config.quickBuild { - nix.package = pkgs.nix_noTests; + nix.package = pkgs.nixComponents.nix-cli; system.forbiddenDependenciesRegexes = [ # This would indicate that the quickBuild feature is broken. @@ -44,4 +44,4 @@ in }; }; }; -} \ No newline at end of file +} From e225b630621712e407d610109d6e866c06e1e948 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 15 Aug 2024 13:51:15 +0200 Subject: [PATCH 138/142] doc: Document function application operator --- doc/manual/src/language/operators.md | 22 ++++++++++++++++++++-- doc/manual/src/language/syntax.md | 2 +- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index 2c2dcc276..ce5b034c6 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -3,7 +3,7 @@ | Name | Syntax | Associativity | Precedence | |----------------------------------------|--------------------------------------------|---------------|------------| | [Attribute selection] | *attrset* `.` *attrpath* \[ `or` *expr* \] | none | 1 | -| Function application | *func* *expr* | left | 2 | +| [Function application] | *func* *expr* | left | 2 | | [Arithmetic negation][arithmetic] | `-` *number* | none | 3 | | [Has attribute] | *attrset* `?` *attrpath* | none | 4 | | List concatenation | *list* `++` *list* | right | 5 | @@ -32,7 +32,7 @@ [string]: ./types.md#type-string [path]: ./types.md#type-path [number]: ./types.md#type-float -[list]: ./types.md#list +[list]: ./types.md#type-list [attribute set]: ./types.md#attribute-set @@ -48,6 +48,22 @@ If the attribute doesn’t exist, return the *expr* after `or` if provided, othe [Attribute selection]: #attribute-selection +## Function application + +> **Syntax** +> +> *func* *expr* + +Apply the callable value *func* to the argument *expr*. Note the absence of a visible operator symbol. +A callable value is either: +- a [user-defined function][function] +- a [built-in][builtins] function +- an attribute set with a [`__functor` attribute](./syntax.md#attr-__functor) + +> **Warning** +> +> This "operator" also separates [list] items, which means that calls in list items must be enclosed by parentheses. + ## Has attribute > **Syntax** @@ -215,3 +231,5 @@ Equivalent to `!`*b1* `||` *b2*. > ``` [Pipe operator]: #pipe-operators +[builtins]: ./builtins.md +[Function application]: #function-application diff --git a/doc/manual/src/language/syntax.md b/doc/manual/src/language/syntax.md index a6a3ea052..506afbea1 100644 --- a/doc/manual/src/language/syntax.md +++ b/doc/manual/src/language/syntax.md @@ -218,7 +218,7 @@ a string), that attribute is simply not added to the set: This will evaluate to `{}` if `foo` evaluates to `false`. -A set that has a `__functor` attribute whose value is callable (i.e. is +A set that has a [`__functor`]{#attr-__functor} attribute whose value is callable (i.e. is itself a function or a set with a `__functor` attribute whose value is callable) can be applied as if it were a function, with the set itself passed in first , e.g., From ce62b766ef976f941b1be8a580afc2c1dc88e17f Mon Sep 17 00:00:00 2001 From: Valentin Gagarin Date: Thu, 15 Aug 2024 15:25:50 +0200 Subject: [PATCH 139/142] fix link from the readme (#11307) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 021e54a3b..ab647e53b 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ Full reference documentation can be found in the [Nix manual](https://nix.dev/re ## Building and developing -Follow instructions in the Nix reference manual to [set up a development environment and build Nix from source](https://nix.dev/manual/nix/development/building.html). +Follow instructions in the Nix reference manual to [set up a development environment and build Nix from source](https://nix.dev/manual/nix/development/development/building.html). ## Contributing From 06b18cff2021d7cca21ad013fea6da0c43109032 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 15 Aug 2024 18:53:42 +0200 Subject: [PATCH 140/142] doc: Edit language/operators Co-authored-by: John Ericson Co-authored-by: Valentin Gagarin --- doc/manual/src/language/operators.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index ce5b034c6..27444258a 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -54,7 +54,7 @@ If the attribute doesn’t exist, return the *expr* after `or` if provided, othe > > *func* *expr* -Apply the callable value *func* to the argument *expr*. Note the absence of a visible operator symbol. +Apply the callable value *func* to the argument *expr*. Note the absence of any visible operator symbol. A callable value is either: - a [user-defined function][function] - a [built-in][builtins] function @@ -62,7 +62,7 @@ A callable value is either: > **Warning** > -> This "operator" also separates [list] items, which means that calls in list items must be enclosed by parentheses. +> [List][list] items are also separated by whitespace, which means that function calls in list items must be enclosed by parentheses. ## Has attribute From 8866d2cd838902d45782541efe08efc1e1f1a2ab Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Fri, 16 Aug 2024 07:09:27 -0700 Subject: [PATCH 141/142] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/63d37ccd2d178d54e7fb691d7ec76000740ea24a?narHash=sha256-7cCC8%2BTdq1%2B3OPyc3%2BgVo9dzUNkNIQfwSDJ2HSi2u3o%3D' (2024-07-21) → 'github:NixOS/nixpkgs/c3d4ac725177c030b1e289015989da2ad9d56af0?narHash=sha256-sqLwJcHYeWLOeP/XoLwAtYjr01TISlkOfz%2BNG82pbdg%3D' (2024-08-15) --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 2ac413a69..b5d0b881c 100644 --- a/flake.lock +++ b/flake.lock @@ -80,11 +80,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1721548954, - "narHash": "sha256-7cCC8+Tdq1+3OPyc3+gVo9dzUNkNIQfwSDJ2HSi2u3o=", + "lastModified": 1723688146, + "narHash": "sha256-sqLwJcHYeWLOeP/XoLwAtYjr01TISlkOfz+NG82pbdg=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "63d37ccd2d178d54e7fb691d7ec76000740ea24a", + "rev": "c3d4ac725177c030b1e289015989da2ad9d56af0", "type": "github" }, "original": { From aa3d35c1f4145c9532620a20d6727c2214eab054 Mon Sep 17 00:00:00 2001 From: Cole Helbling Date: Fri, 16 Aug 2024 07:22:30 -0700 Subject: [PATCH 142/142] ci: check that all outputs for all systems can evaluate --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9831d0e0e..3463335b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,6 +49,7 @@ jobs: done ) & - run: nix --experimental-features 'nix-command flakes' flake check -L + - run: nix --experimental-features 'nix-command flakes' flake show --all-systems --json # Steps to test CI automation in your own fork. # Cachix: