diff --git a/.clang-format b/.clang-format index 3067583e1..4f191fc18 100644 --- a/.clang-format +++ b/.clang-format @@ -31,3 +31,4 @@ AlwaysBreakBeforeMultilineStrings: true IndentPPDirectives: AfterHash PPIndentWidth: 2 BinPackArguments: false +BreakBeforeTernaryOperators: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4eb9cf10d..3463335b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,6 +49,7 @@ jobs: done ) & - run: nix --experimental-features 'nix-command flakes' flake check -L + - run: nix --experimental-features 'nix-command flakes' flake show --all-systems --json # Steps to test CI automation in your own fork. # Cachix: @@ -195,20 +196,6 @@ jobs: - uses: DeterminateSystems/magic-nix-cache-action@main - run: nix build -L .#hydraJobs.tests.githubFlakes .#hydraJobs.tests.tarballFlakes .#hydraJobs.tests.functional_user - meson_build: - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - uses: DeterminateSystems/nix-installer-action@main - - uses: DeterminateSystems/magic-nix-cache-action@main - # Only meson packages that don't have a tests.run derivation. - # Those that have it are already built and tested as part of nix flake check. - - run: nix build -L .#hydraJobs.build.{nix-cmd,nix-main}.$(nix-instantiate --eval --expr builtins.currentSystem | sed -e 's/"//g') - flake_regressions: needs: vm_tests runs-on: ubuntu-22.04 diff --git a/.version b/.version index ad2261920..5c18f9195 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.24.0 +2.25.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 38f5d43b7..da56653b8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,9 +41,9 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). There are many open pull requests that might already do what you intend to work on. You can use [labels](https://github.com/NixOS/nix/labels) to filter for relevant topics. -3. Check the [Nix reference manual](https://nixos.org/manual/nix/unstable/contributing/hacking.html) for information on building Nix and running its tests. +3. Check the [Nix reference manual](https://nix.dev/manual/nix/development/development/building.html) for information on building Nix and running its tests. - For contributions to the command line interface, please check the [CLI guidelines](https://nixos.org/manual/nix/unstable/contributing/cli-guideline.html). + For contributions to the command line interface, please check the [CLI guidelines](https://nix.dev/manual/nix/development/development/cli-guideline.html). 4. Make your change! @@ -52,6 +52,20 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). Link related issues to inform interested parties and future contributors about your change. If your pull request closes one or multiple issues, mention that in the description using `Closes: #`, as it will then happen automatically when your change is merged. + * Credit original authors when you're reusing or building on their work. + * Link to relevant changes in other projects, so that others can understand the full context of the change in the future when you or someone else will change or troubleshoot the code. + This is especially important when your change is based on work done in other repositories. + + Example: + ``` + This is based on the work of @user in . + This solution took inspiration from . + + Co-authored-by: User Name + ``` + + When cherry-picking from a different repository, use the `-x` flag, and then amend the commits to turn the hashes into URLs. + * Make sure to have [a clean history of commits on your branch by using rebase](https://www.digitalocean.com/community/tutorials/how-to-rebase-and-update-a-pull-request). * [Mark the pull request as draft](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/changing-the-stage-of-a-pull-request) if you're not done with the changes. @@ -69,7 +83,7 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). - [ ] API documentation in header files - [ ] Code and comments are self-explanatory - [ ] Commit message explains **why** the change was made - - [ ] New feature or incompatible change: [add a release note](https://nixos.org/manual/nix/stable/contributing/hacking#add-a-release-note) + - [ ] New feature or incompatible change: [add a release note](https://nix.dev/manual/nix/development/development/contributing.html#add-a-release-note) 7. If you need additional feedback or help to getting pull request into shape, ask other contributors using [@mentions](https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax#mentioning-people-and-teams). @@ -78,7 +92,7 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). The Nix reference manual is hosted on https://nixos.org/manual/nix. The underlying source files are located in [`doc/manual/src`](./doc/manual/src). For small changes you can [use GitHub to edit these files](https://docs.github.com/en/repositories/working-with-files/managing-files/editing-files) -For larger changes see the [Nix reference manual](https://nixos.org/manual/nix/unstable/contributing/hacking.html). +For larger changes see the [Nix reference manual](https://nix.dev/manual/nix/development/development/contributing.html). ## Getting help diff --git a/Makefile b/Makefile index bb64a104e..b51ae6cc7 100644 --- a/Makefile +++ b/Makefile @@ -38,22 +38,11 @@ makefiles += \ endif endif -ifeq ($(ENABLE_UNIT_TESTS), yes) -makefiles += \ - tests/unit/libutil/local.mk \ - tests/unit/libutil-support/local.mk \ - tests/unit/libstore/local.mk \ - tests/unit/libstore-support/local.mk \ - tests/unit/libfetchers/local.mk \ - tests/unit/libexpr/local.mk \ - tests/unit/libexpr-support/local.mk \ - tests/unit/libflake/local.mk -endif - ifeq ($(ENABLE_FUNCTIONAL_TESTS), yes) ifdef HOST_UNIX makefiles += \ tests/functional/local.mk \ + tests/functional/flakes/local.mk \ tests/functional/ca/local.mk \ tests/functional/git-hashing/local.mk \ tests/functional/dyn-drv/local.mk \ @@ -92,7 +81,7 @@ ifdef HOST_WINDOWS GLOBAL_LDFLAGS += -Wl,--export-all-symbols endif -GLOBAL_CXXFLAGS += -g -Wall -Wdeprecated-copy -Wignored-qualifiers -Wimplicit-fallthrough -Werror=unused-result -include $(buildprefix)config.h -std=c++2a -I src +GLOBAL_CXXFLAGS += -g -Wall -Wdeprecated-copy -Wignored-qualifiers -Wimplicit-fallthrough -Werror=unused-result -Werror=suggest-override -include $(buildprefix)config.h -std=c++2a -I src # Include the main lib, causing rules to be defined @@ -103,13 +92,6 @@ include mk/lib.mk # These must be defined after `mk/lib.mk`. Otherwise the first rule # incorrectly becomes the default target. -ifneq ($(ENABLE_UNIT_TESTS), yes) -.PHONY: check -check: - @echo "Unit tests are disabled. Configure without '--disable-unit-tests', or avoid calling 'make check'." - @exit 1 -endif - ifneq ($(ENABLE_FUNCTIONAL_TESTS), yes) .PHONY: installcheck installcheck: diff --git a/Makefile.config.in b/Makefile.config.in index 3100d2073..e131484f6 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -12,7 +12,6 @@ ENABLE_BUILD = @ENABLE_BUILD@ ENABLE_DOC_GEN = @ENABLE_DOC_GEN@ ENABLE_FUNCTIONAL_TESTS = @ENABLE_FUNCTIONAL_TESTS@ ENABLE_S3 = @ENABLE_S3@ -ENABLE_UNIT_TESTS = @ENABLE_UNIT_TESTS@ GTEST_LIBS = @GTEST_LIBS@ HAVE_LIBCPUID = @HAVE_LIBCPUID@ HAVE_SECCOMP = @HAVE_SECCOMP@ diff --git a/build-utils-meson/diagnostics/meson.build b/build-utils-meson/diagnostics/meson.build index 2b79f6566..30eedfc13 100644 --- a/build-utils-meson/diagnostics/meson.build +++ b/build-utils-meson/diagnostics/meson.build @@ -1,13 +1,11 @@ add_project_arguments( - '-Wno-deprecated-declarations', - '-Wimplicit-fallthrough', + '-Wdeprecated-copy', + '-Werror=suggest-override', '-Werror=switch', '-Werror=switch-enum', '-Werror=unused-result', - '-Wdeprecated-copy', '-Wignored-qualifiers', - # Enable assertions in libstdc++ by default. Harmless on libc++. Benchmarked - # at ~1% overhead in `nix search`. - # + '-Wimplicit-fallthrough', + '-Wno-deprecated-declarations', language : 'cpp', ) diff --git a/configure.ac b/configure.ac index 5c22ed176..18d718c07 100644 --- a/configure.ac +++ b/configure.ac @@ -141,18 +141,6 @@ AC_ARG_ENABLE(build, AS_HELP_STRING([--disable-build],[Do not build nix]), ENABLE_BUILD=$enableval, ENABLE_BUILD=yes) AC_SUBST(ENABLE_BUILD) -# Building without unit tests is useful for bootstrapping with a smaller footprint -# or running the tests in a separate derivation. Otherwise, we do compile and -# run them. - -AC_ARG_ENABLE(unit-tests, AS_HELP_STRING([--disable-unit-tests],[Do not build the tests]), - ENABLE_UNIT_TESTS=$enableval, ENABLE_UNIT_TESTS=$ENABLE_BUILD) -AC_SUBST(ENABLE_UNIT_TESTS) - -AS_IF( - [test "$ENABLE_BUILD" == "no" && test "$ENABLE_UNIT_TESTS" == "yes"], - [AC_MSG_ERROR([Cannot enable unit tests when building overall is disabled. Please do not pass '--enable-unit-tests' or do not pass '--disable-build'.])]) - AC_ARG_ENABLE(functional-tests, AS_HELP_STRING([--disable-functional-tests],[Do not build the tests]), ENABLE_FUNCTIONAL_TESTS=$enableval, ENABLE_FUNCTIONAL_TESTS=yes) AC_SUBST(ENABLE_FUNCTIONAL_TESTS) @@ -358,16 +346,6 @@ if test "$gc" = yes; then CFLAGS="$old_CFLAGS" fi -AS_IF([test "$ENABLE_UNIT_TESTS" == "yes"],[ - -# Look for gtest. -PKG_CHECK_MODULES([GTEST], [gtest_main gmock_main]) - -# Look for rapidcheck. -PKG_CHECK_MODULES([RAPIDCHECK], [rapidcheck rapidcheck_gtest]) - -]) - # Look for nlohmann/json. PKG_CHECK_MODULES([NLOHMANN_JSON], [nlohmann_json >= 3.9]) diff --git a/doc/manual/custom.css b/doc/manual/custom.css index 9e8e3886f..7af150be3 100644 --- a/doc/manual/custom.css +++ b/doc/manual/custom.css @@ -12,8 +12,8 @@ h1.menu-title::before { } -h1.menu-title { - padding: 0.5em; +.menu-bar { + padding: 0.5em 0em; } .sidebar .sidebar-scrollbox { diff --git a/doc/manual/generate-builtins.nix b/doc/manual/generate-builtins.nix index 13de6c397..37ed12a43 100644 --- a/doc/manual/generate-builtins.nix +++ b/doc/manual/generate-builtins.nix @@ -12,7 +12,7 @@ let experimentalNotice = optionalString (experimental-feature != null) '' > **Note** > - > This function is only available if the [`${experimental-feature}` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-${experimental-feature}) is enabled. + > This function is only available if the [`${experimental-feature}` experimental feature](@docroot@/development/experimental-features.md#xp-feature-${experimental-feature}) is enabled. > > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): > diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 90eaa1a73..791bfd2c7 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -38,7 +38,7 @@ let result = '' > **Warning** \ > This program is - > [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) + > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) > and its interface is subject to change. # Name diff --git a/doc/manual/generate-settings.nix b/doc/manual/generate-settings.nix index 504cda362..93a8e093e 100644 --- a/doc/manual/generate-settings.nix +++ b/doc/manual/generate-settings.nix @@ -33,10 +33,10 @@ let > **Warning** > > This setting is part of an - > [experimental feature](@docroot@/contributing/experimental-features.md). + > [experimental feature](@docroot@/development/experimental-features.md). > > To change this setting, make sure the - > [`${experimentalFeature}` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}) + > [`${experimentalFeature}` experimental feature](@docroot@/development/experimental-features.md#xp-feature-${experimentalFeature}) > is enabled. > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): > diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix index c311c3c39..cc3704124 100644 --- a/doc/manual/generate-store-info.nix +++ b/doc/manual/generate-store-info.nix @@ -32,10 +32,10 @@ let > **Warning** > > This store is part of an - > [experimental feature](@docroot@/contributing/experimental-features.md). + > [experimental feature](@docroot@/development/experimental-features.md). > > To use this store, make sure the - > [`${experimentalFeature}` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}) + > [`${experimentalFeature}` experimental feature](@docroot@/development/experimental-features.md#xp-feature-${experimentalFeature}) > is enabled. > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): > diff --git a/doc/manual/generate-xp-features-shortlist.nix b/doc/manual/generate-xp-features-shortlist.nix index ec09f4b75..eb735ba5f 100644 --- a/doc/manual/generate-xp-features-shortlist.nix +++ b/doc/manual/generate-xp-features-shortlist.nix @@ -4,6 +4,6 @@ with import ; let showExperimentalFeature = name: doc: '' - - [`${name}`](@docroot@/contributing/experimental-features.md#xp-feature-${name}) + - [`${name}`](@docroot@/development/experimental-features.md#xp-feature-${name}) ''; in xps: indent " " (concatStrings (attrValues (mapAttrs showExperimentalFeature xps))) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 0cec52885..fcc50f460 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -95,7 +95,7 @@ $(d)/nix-profiles.5: $(d)/src/command-ref/files/profiles.md $(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@ @rm $^.tmp -$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md +$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/development/experimental-feature-descriptions.md @cp $< $@ @$(call process-includes,$@,$@) @@ -124,7 +124,7 @@ $(d)/conf-file.json: $(doc_nix) $(trace-gen) $(dummy-env) $(doc_nix) config show --json --experimental-features nix-command > $@.tmp @mv $@.tmp $@ -$(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(doc_nix) +$(d)/src/development/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(doc_nix) @rm -rf $@ $@.tmp $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features.nix (builtins.fromJSON (builtins.readFile $<))' @mv $@.tmp $@ @@ -207,11 +207,11 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli done @touch $@ -# the `! -name 'contributing.md'` filter excludes the one place where +# the `! -name 'documentation.md'` filter excludes the one place where # `@docroot@` is to be preserved for documenting the mechanism # FIXME: maybe contributing guides should live right next to the code # instead of in the manual -$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/release-notes/rl-next.md $(d)/src/figures $(d)/src/favicon.png $(d)/src/favicon.svg +$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/development/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/release-notes/rl-next.md $(d)/src/figures $(d)/src/favicon.png $(d)/src/favicon.svg $(trace-gen) \ tmp="$$(mktemp -d)"; \ cp -r doc/manual "$$tmp"; \ diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index 0f9f91b03..cb8cd18fa 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -143,7 +143,7 @@ const redirects = { "opt-timeout": "command-ref/opt-common.html#opt-timeout", "sec-common-options": "command-ref/opt-common.html", "ch-utilities": "command-ref/utilities.html", - "chap-hacking": "contributing/hacking.html", + "chap-hacking": "development/building.html", "adv-attr-allowSubstitutes": "language/advanced-attributes.html#adv-attr-allowSubstitutes", "adv-attr-allowedReferences": "language/advanced-attributes.html#adv-attr-allowedReferences", "adv-attr-allowedRequisites": "language/advanced-attributes.html#adv-attr-allowedRequisites", @@ -344,13 +344,14 @@ const redirects = { }, "language/syntax.html": { "scoping-rules": "scoping.html", + "string-literal": "string-literals.html", }, "installation/installing-binary.html": { "linux": "uninstall.html#linux", "macos": "uninstall.html#macos", "uninstalling": "uninstall.html", }, - "contributing/hacking.html": { + "development/building.html": { "nix-with-flakes": "#building-nix-with-flakes", "classic-nix": "#building-nix", "running-tests": "testing.html#running-tests", @@ -361,7 +362,12 @@ const redirects = { "installer-tests": "testing.html#installer-tests", "one-time-setup": "testing.html#one-time-setup", "using-the-ci-generated-installer-for-manual-testing": "testing.html#using-the-ci-generated-installer-for-manual-testing", - "characterization-testing": "#characterisation-testing-unit", + "characterization-testing": "testing.html#characterisation-testing-unit", + "add-a-release-note": "contributing.html#add-a-release-note", + "add-an-entry": "contributing.html#add-an-entry", + "build-process": "contributing.html#build-process", + "reverting": "contributing.html#reverting", + "branches": "contributing.html#branches", }, "glossary.html": { "gloss-local-store": "store/types/local-store.html", diff --git a/doc/manual/rl-next/ban-integer-overflow.md b/doc/manual/rl-next/ban-integer-overflow.md new file mode 100644 index 000000000..0e553af76 --- /dev/null +++ b/doc/manual/rl-next/ban-integer-overflow.md @@ -0,0 +1,21 @@ +--- +synopsis: Define integer overflow in the Nix language as an error +issues: [10968] +prs: [11188] +--- + +Previously, integer overflow in the Nix language invoked C++ level signed overflow, which was undefined behaviour, but *usually* manifested as wrapping around on overflow. + +Since prior to the public release of Lix, Lix had C++ signed overflow defined to crash the process and nobody noticed this having accidentally removed overflow from the Nix language for three months until it was caught by fiddling around. +Given the significant body of actual Nix code that has been evaluated by Lix in that time, it does not appear that nixpkgs or much of importance depends on integer overflow, so it appears safe to turn into an error. + +Some other overflows were fixed: +- `builtins.fromJSON` of values greater than the maximum representable value in a signed 64-bit integer will generate an error. +- `nixConfig` in flakes will no longer accept negative values for configuration options. + +Integer overflow now looks like the following: + +``` +$ nix eval --expr '9223372036854775807 + 1' +error: integer overflow in adding 9223372036854775807 + 1 +``` diff --git a/doc/manual/rl-next/build-hook-default.md b/doc/manual/rl-next/build-hook-default.md new file mode 100644 index 000000000..197290536 --- /dev/null +++ b/doc/manual/rl-next/build-hook-default.md @@ -0,0 +1,22 @@ +--- +synopsis: |- + The `build-hook` setting's default is less useful when using `libnixstore` as a library +prs: +- 11178 +--- + +*This is an obscure issue that only affects usage of the `libnixstore` library outside of the Nix executable.* + +As part the ongoing [rewrite of the build system](https://github.com/NixOS/nix/issues/2503) to use [Meson](https://mesonbuild.com/), we are also switching to packaging individual Nix components separately (and building them in separate derivations). +This means that when building `libnixstore` we do not know where the Nix binaries will be installed --- `libnixstore` doesn't know about downstream consumers like the Nix binaries at all. + +*This is also unrelated to the _`post`_-`build-hook`*, which is often used for pushing to a cache.* + +This has a small adverse affect on remote building --- the `build-remote` executable that is specified from the [`build-hook`](@docroot@/command-ref/conf-file.md#conf-build-hook) setting will not be gotten from the (presumed) installation location, but instead looked up on the `PATH`. +This means that other applications linking `libnixstore` that wish to use remote building must arrange for the `nix` command to be on the PATH (or manually overriding `build-hook`) in order for that to work. + +Long term we don't envision this being a downside, because we plan to [get rid of `build-remote` and the build hook setting entirely](https://github.com/NixOS/nix/issues/1221). +There is simply no need to add a second layer of remote-procedure-calling when we want to connect to a remote builder. +The build hook protocol did in principle support custom ways of remote building, but that can also be accomplished with a custom service for the ssh or daemon/ssh-ng protocols, or with a custom [store type](@docroot@/store/types/) i.e. `Store` subclass. + +The Perl bindings no longer expose `getBinDir` either, since they libraries those bindings wrap no longer know the location of installed binaries as described above. diff --git a/doc/manual/rl-next/drop-vendored-toml11.md b/doc/manual/rl-next/drop-vendored-toml11.md deleted file mode 100644 index d1feeb703..000000000 --- a/doc/manual/rl-next/drop-vendored-toml11.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -synopsis: Stop vendoring toml11 ---- - -We don't apply any patches to it, and vendoring it locks users into -bugs (it hasn't been updated since its introduction in late 2021). diff --git a/doc/manual/rl-next/harden-user-sandboxing.md b/doc/manual/rl-next/harden-user-sandboxing.md deleted file mode 100644 index a647acf25..000000000 --- a/doc/manual/rl-next/harden-user-sandboxing.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -synopsis: Harden the user sandboxing -significance: significant -issues: ---- - -The build directory has been hardened against interference with the outside world by nesting it inside another directory owned by (and only readable by) the daemon user. diff --git a/doc/manual/rl-next/nix-shell-looks-for-shell-nix.md b/doc/manual/rl-next/nix-shell-looks-for-shell-nix.md deleted file mode 100644 index 99be4148b..000000000 --- a/doc/manual/rl-next/nix-shell-looks-for-shell-nix.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -synopsis: "`nix-shell ` looks for `shell.nix`" -significance: significant -issues: -- 496 -- 2279 -- 4529 -- 5431 -- 11053 -prs: -- 11057 ---- - -`nix-shell $x` now looks for `$x/shell.nix` when `$x` resolves to a directory. - -Although this might be seen as a breaking change, its primarily interactive usage makes it a minor issue. -This adjustment addresses a commonly reported problem. - -This also applies to `nix-shell` shebang scripts. Consider the following example: - -```shell -#!/usr/bin/env nix-shell -#!nix-shell -i bash -``` - -This will now load `shell.nix` from the script's directory, if it exists; `default.nix` otherwise. - -The old behavior can be opted into by setting the option [`nix-shell-always-looks-for-shell-nix`](@docroot@/command-ref/conf-file.md#conf-nix-shell-always-looks-for-shell-nix) to `false`. diff --git a/doc/manual/rl-next/repl-doc-renders-doc-comments.md b/doc/manual/rl-next/repl-doc-renders-doc-comments.md deleted file mode 100644 index 05023697c..000000000 --- a/doc/manual/rl-next/repl-doc-renders-doc-comments.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -synopsis: "`nix-repl`'s `:doc` shows documentation comments" -significance: significant -issues: -- 3904 -- 10771 -prs: -- 1652 -- 9054 -- 11072 ---- - -`nix repl` has a `:doc` command that previously only rendered documentation for internally defined functions. -This feature has been extended to also render function documentation comments, in accordance with [RFC 145]. - -Example: - -``` -nix-repl> :doc lib.toFunction -Function toFunction - … defined at /home/user/h/nixpkgs/lib/trivial.nix:1072:5 - - Turns any non-callable values into constant functions. Returns - callable values as is. - -Inputs - - v - - : Any value - -Examples - - :::{.example} - -## lib.trivial.toFunction usage example - - | nix-repl> lib.toFunction 1 2 - | 1 - | - | nix-repl> lib.toFunction (x: x + 1) 2 - | 3 - - ::: -``` - -Known limitations: -- It does not render documentation for "formals", such as `{ /** the value to return */ x, ... }: x`. -- Some extensions to markdown are not yet supported, as you can see in the example above. - -We'd like to acknowledge Yingchi Long for proposing a proof of concept for this functionality in [#9054](https://github.com/NixOS/nix/pull/9054), as well as @sternenseemann and Johannes Kirschbauer for their contributions, proposals, and their work on [RFC 145]. - -[RFC 145]: https://github.com/NixOS/rfcs/pull/145 diff --git a/doc/manual/rl-next/shebang-relative.md b/doc/manual/rl-next/shebang-relative.md deleted file mode 100644 index c887a598a..000000000 --- a/doc/manual/rl-next/shebang-relative.md +++ /dev/null @@ -1,62 +0,0 @@ ---- -synopsis: "`nix-shell` shebang uses relative path" -prs: -- 5088 -- 11058 -issues: -- 4232 ---- - - -Relative [path](@docroot@/language/values.md#type-path) literals in `nix-shell` shebang scripts' options are now resolved relative to the [script's location](@docroot@/glossary?highlight=base%20directory#gloss-base-directory). -Previously they were resolved relative to the current working directory. - -For example, consider the following script in `~/myproject/say-hi`: - -```shell -#!/usr/bin/env nix-shell -#!nix-shell --expr 'import ./shell.nix' -#!nix-shell --arg toolset './greeting-tools.nix' -#!nix-shell -i bash -hello -``` - -Older versions of `nix-shell` would resolve `shell.nix` relative to the current working directory; home in this example: - -```console -[hostname:~]$ ./myproject/say-hi -error: - … while calling the 'import' builtin - at «string»:1:2: - 1| (import ./shell.nix) - | ^ - - error: path '/home/user/shell.nix' does not exist -``` - -Since this release, `nix-shell` resolves `shell.nix` relative to the script's location, and `~/myproject/shell.nix` is used. - -```console -$ ./myproject/say-hi -Hello, world! -``` - -**Opt-out** - -This is technically a breaking change, so we have added an option so you can adapt independently of your Nix update. -The old behavior can be opted into by setting the option [`nix-shell-shebang-arguments-relative-to-script`](@docroot@/command-ref/conf-file.md#conf-nix-shell-shebang-arguments-relative-to-script) to `false`. -This option will be removed in a future release. - -**`nix` command shebang** - -The experimental [`nix` command shebang](@docroot@/command-ref/new-cli/nix.md?highlight=shebang#shebang-interpreter) already behaves in this script-relative manner. - -Example: - -```shell -#!/usr/bin/env nix -#!nix develop -#!nix --expr ``import ./shell.nix`` -#!nix -c bash -hello -``` diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index a6a2101e9..eef7d189c 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -28,6 +28,9 @@ - [Data Types](language/types.md) - [String context](language/string-context.md) - [Syntax and semantics](language/syntax.md) + - [Variables](language/variables.md) + - [String literals](language/string-literals.md) + - [Identifiers](language/identifiers.md) - [Scoping rules](language/scope.md) - [String interpolation](language/string-interpolation.md) - [Lookup path](language/constructs/lookup-path.md) @@ -115,16 +118,18 @@ - [Derivation "ATerm" file format](protocols/derivation-aterm.md) - [C API](c-api.md) - [Glossary](glossary.md) -- [Contributing](contributing/index.md) - - [Hacking](contributing/hacking.md) - - [Testing](contributing/testing.md) - - [Documentation](contributing/documentation.md) - - [Experimental Features](contributing/experimental-features.md) - - [CLI guideline](contributing/cli-guideline.md) - - [JSON guideline](contributing/json-guideline.md) - - [C++ style guide](contributing/cxx.md) +- [Development](development/index.md) + - [Building](development/building.md) + - [Testing](development/testing.md) + - [Documentation](development/documentation.md) + - [CLI guideline](development/cli-guideline.md) + - [JSON guideline](development/json-guideline.md) + - [C++ style guide](development/cxx.md) + - [Experimental Features](development/experimental-features.md) + - [Contributing](development/contributing.md) - [Releases](release-notes/index.md) {{#include ./SUMMARY-rl-next.md}} + - [Release 2.24 (2024-07-31)](release-notes/rl-2.24.md) - [Release 2.23 (2024-06-03)](release-notes/rl-2.23.md) - [Release 2.22 (2024-04-23)](release-notes/rl-2.22.md) - [Release 2.21 (2024-03-11)](release-notes/rl-2.21.md) diff --git a/doc/manual/src/_redirects b/doc/manual/src/_redirects index 578c48f06..07b3130f9 100644 --- a/doc/manual/src/_redirects +++ b/doc/manual/src/_redirects @@ -20,7 +20,15 @@ /command-ref/command-ref /command-ref 301! -/contributing/contributing /contributing 301! +/contributing/contributing /development 301! +/contributing /development 301! +/contributing/hacking /development/building 301! +/contributing/testing /development/testing 301! +/contributing/documentation /development/documentation 301! +/contributing/experimental-features /development/experimental-features 301! +/contributing/cli-guideline /development/cli-guideline 301! +/contributing/json-guideline /development/json-guideline 301! +/contributing/cxx /development/cxx 301! /expressions/expression-language /language/ 301! /expressions/language-constructs /language/constructs 301! diff --git a/doc/manual/src/c-api.md b/doc/manual/src/c-api.md index 29df0b644..0cdd83832 100644 --- a/doc/manual/src/c-api.md +++ b/doc/manual/src/c-api.md @@ -10,7 +10,7 @@ See: - [Matrix Room *Nix Bindings*](https://matrix.to/#/#nix-bindings:nixos.org) for discussion and questions. - [Stabilisation Milestone](https://github.com/NixOS/nix/milestone/52) - [Other C API PRs and issues](https://github.com/NixOS/nix/labels/c%20api) -- [Contributing C API Documentation](contributing/documentation.md#c-api-documentation), including how to build it locally. +- [Contributing C API Documentation](development/documentation.md#c-api-documentation), including how to build it locally. [Getting Started]: https://hydra.nixos.org/job/nix/master/external-api-docs/latest/download-by-type/doc/external-api-docs [Index]: https://hydra.nixos.org/job/nix/master/external-api-docs/latest/download-by-type/doc/external-api-docs/globals.html diff --git a/doc/manual/src/command-ref/env-common.md b/doc/manual/src/command-ref/env-common.md index d3f5f9c14..0b5017882 100644 --- a/doc/manual/src/command-ref/env-common.md +++ b/doc/manual/src/command-ref/env-common.md @@ -9,22 +9,26 @@ Most Nix commands interpret the following environment variables: - [`NIX_PATH`](#env-NIX_PATH) - A colon-separated list of directories used to look up the location of Nix - expressions using [paths](@docroot@/language/types.md#type-path) - enclosed in angle brackets (i.e., ``), - e.g. `/home/eelco/Dev:/etc/nixos`. It can be extended using the - [`-I` option](@docroot@/command-ref/opt-common.md#opt-I). + A colon-separated list of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). - If `NIX_PATH` is not set at all, Nix will fall back to the following list in [impure](@docroot@/command-ref/conf-file.md#conf-pure-eval) and [unrestricted](@docroot@/command-ref/conf-file.md#conf-restrict-eval) evaluation mode: + This environment variable overrides the value of the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path). - 1. `$HOME/.nix-defexpr/channels` - 2. `nixpkgs=/nix/var/nix/profiles/per-user/root/channels/nixpkgs` - 3. `/nix/var/nix/profiles/per-user/root/channels` + It can be extended using the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I). + + > **Example** + > + > ```bash + > $ export NIX_PATH=`/home/eelco/Dev:nixos-config=/etc/nixos + > ``` If `NIX_PATH` is set to an empty string, resolving search paths will always fail. - For example, attempting to use `` will produce: - error: file 'nixpkgs' was not found in the Nix search path + > **Example** + > + > ```bash + > $ NIX_PATH= nix-instantiate --eval '' + > error: file 'nixpkgs' was not found in the Nix search path (add it using $NIX_PATH or -I) + > ``` - [`NIX_IGNORE_SYMLINK_STORE`](#env-NIX_IGNORE_SYMLINK_STORE) diff --git a/doc/manual/src/command-ref/experimental-commands.md b/doc/manual/src/command-ref/experimental-commands.md index 286ddc6d6..1190729a2 100644 --- a/doc/manual/src/command-ref/experimental-commands.md +++ b/doc/manual/src/command-ref/experimental-commands.md @@ -1,6 +1,6 @@ # Experimental Commands -This section lists [experimental commands](@docroot@/contributing/experimental-features.md#xp-feature-nix-command). +This section lists [experimental commands](@docroot@/development/experimental-features.md#xp-feature-nix-command). > **Warning** > diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index ddec30f5b..69a711bd5 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -297,3 +297,8 @@ with import {}; runCommand "dummy" { buildInputs = [ python pythonPackages.prettytable ]; } "" ``` + +The script's file name is passed as the first argument to the interpreter specified by the `-i` flag. + +Aside from the very first line, which is a directive to the operating system, the additional `#! nix-shell` lines do not need to be at the beginning of the file. +This allows wrapping them in block comments for languages where `#` does not start a comment, such as ECMAScript, Erlang, PHP, or Ruby. diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index e30b351a4..a899758df 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -32,7 +32,7 @@ If no substitutes are available and no store derivation is given, realisation fa [store objects]: @docroot@/store/store-object.md [closure]: @docroot@/glossary.md#gloss-closure [substituters]: @docroot@/command-ref/conf-file.md#conf-substituters -[content-addressed derivations]: @docroot@/contributing/experimental-features.md#xp-feature-ca-derivations +[content-addressed derivations]: @docroot@/development/experimental-features.md#xp-feature-ca-derivations [Nix database]: @docroot@/glossary.md#gloss-nix-database The resulting paths are printed on standard output. diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index a42909e2d..69a700207 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -37,7 +37,7 @@ Most Nix commands accept the following command-line options: Print even more informational messages. - `4` “Debug” - + Print debug information. - `5` “Vomit” @@ -187,11 +187,12 @@ Most Nix commands accept the following command-line options: For `nix-shell`, this option is commonly used to give you a shell in which you can build the packages returned by the expression. If you want to get a shell which contain the *built* packages ready for use, give your expression to the `nix-shell --packages ` convenience flag instead. -- [`-I`](#opt-I) *path* +- [`-I` / `--include`](#opt-I) *path* - Add an entry to the [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path). + Add an entry to the list of search paths used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). This option may be given multiple times. - Paths added through `-I` take precedence over [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH). + + Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH). - [`--option`](#opt-option) *name* *value* diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/development/building.md similarity index 74% rename from doc/manual/src/contributing/hacking.md rename to doc/manual/src/development/building.md index c128515e9..5a5fb3368 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/development/building.md @@ -1,24 +1,67 @@ -# Hacking +# Building Nix -This section provides some notes on how to hack on Nix. To get the -latest version of Nix from GitHub: +This section provides some notes on how to start hacking on Nix. +To get the latest version of Nix from GitHub: ```console $ git clone https://github.com/NixOS/nix.git $ cd nix ``` -The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. If you don't have it installed, follow the [installation instructions]. +> **Note** +> +> The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. +> If you don't have it installed, follow the [installation instructions](../installation/index.md). -[installation instructions]: ../installation/index.md + +To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: + +```console +$ nix-shell +``` + +To get a shell with one of the other [supported compilation environments](#compilation-environments): + +```console +$ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages +``` + +> **Note** +> +> You can use `native-ccacheStdenvPackages` to drastically improve rebuild time. +> By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`. + +To build Nix itself in this shell: + +```console +[nix-shell]$ autoreconfPhase +[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out +[nix-shell]$ make -j $NIX_BUILD_CORES +``` + +To install it in `$(pwd)/outputs` and test it: + +```console +[nix-shell]$ make install +[nix-shell]$ make installcheck -j $NIX_BUILD_CORES +[nix-shell]$ ./outputs/out/bin/nix --version +nix (Nix) 2.12 +``` + +To build a release version of Nix for the current operating system and CPU architecture: + +```console +$ nix-build +``` + +You can also build Nix for one of the [supported platforms](#platforms). ## Building Nix with flakes This section assumes you are using Nix with the [`flakes`] and [`nix-command`] experimental features enabled. -See the [Building Nix](#building-nix) section for equivalent instructions using stable Nix interfaces. -[`flakes`]: @docroot@/contributing/experimental-features.md#xp-feature-flakes -[`nix-command`]: @docroot@/contributing/experimental-features.md#xp-nix-command +[`flakes`]: @docroot@/development/experimental-features.md#xp-feature-flakes +[`nix-command`]: @docroot@/development/experimental-features.md#xp-nix-command To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: @@ -67,50 +110,6 @@ $ nix build You can also build Nix for one of the [supported platforms](#platforms). -## Building Nix - -To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: - -```console -$ nix-shell -``` - -To get a shell with one of the other [supported compilation environments](#compilation-environments): - -```console -$ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages -``` - -> **Note** -> -> You can use `native-ccacheStdenvPackages` to drastically improve rebuild time. -> By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`. - -To build Nix itself in this shell: - -```console -[nix-shell]$ autoreconfPhase -[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out -[nix-shell]$ make -j $NIX_BUILD_CORES -``` - -To install it in `$(pwd)/outputs` and test it: - -```console -[nix-shell]$ make install -[nix-shell]$ make installcheck -j $NIX_BUILD_CORES -[nix-shell]$ ./outputs/out/bin/nix --version -nix (Nix) 2.12 -``` - -To build a release version of Nix for the current operating system and CPU architecture: - -```console -$ nix-build -``` - -You can also build Nix for one of the [supported platforms](#platforms). - ## Makefile variables You may need `profiledir=$out/etc/profile.d` and `sysconfdir=$out/etc` to run `make install`. @@ -294,81 +293,3 @@ If it fails, run `git add --patch` to approve the suggestions _and commit again_ To refresh pre-commit hook's config file, do the following: 1. Exit the development shell and start it again by running `nix develop`. 2. If you also use the pre-commit hook, also run `pre-commit-hooks-install` again. - -## Add a release note - -`doc/manual/rl-next` contains release notes entries for all unreleased changes. - -User-visible changes should come with a release note. - -### Add an entry - -Here's what a complete entry looks like. The file name is not incorporated in the document. - -``` ---- -synopsis: Basically a title -issues: 1234 -prs: 1238 ---- - -Here's one or more paragraphs that describe the change. - -- It's markdown -- Add references to the manual using @docroot@ -``` - -Significant changes should add the following header, which moves them to the top. - -``` -significance: significant -``` - - -See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog). - -### Build process - -Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. - -## Branches - -- [`master`](https://github.com/NixOS/nix/commits/master) - - The main development branch. All changes are approved and merged here. - When developing a change, create a branch based on the latest `master`. - - Maintainers try to [keep it in a release-worthy state](#reverting). - -- [`maintenance-*.*`](https://github.com/NixOS/nix/branches/all?query=maintenance) - - These branches are the subject of backports only, and are - also [kept](#reverting) in a release-worthy state. - - See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) - -- [`latest-release`](https://github.com/NixOS/nix/tree/latest-release) - - The latest patch release of the latest minor version. - - See [`maintainers/release-process.md`](https://github.com/NixOS/nix/blob/master/maintainers/release-process.md) - -- [`backport-*-to-*`](https://github.com/NixOS/nix/branches/all?query=backport) - - Generally branches created by the backport action. - - See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) - -- [_other_](https://github.com/NixOS/nix/branches/all) - - Branches that do not conform to the above patterns should be feature branches. - -## Reverting - -If a change turns out to be merged by mistake, or contain a regression, it may be reverted. -A revert is not a rejection of the contribution, but merely part of an effective development process. -It makes sure that development keeps running smoothly, with minimal uncertainty, and less overhead. -If maintainers have to worry too much about avoiding reverts, they would not be able to merge as much. -By embracing reverts as a good part of the development process, everyone wins. - -However, taking a step back may be frustrating, so maintainers will be extra supportive on the next try. diff --git a/doc/manual/src/contributing/cli-guideline.md b/doc/manual/src/development/cli-guideline.md similarity index 100% rename from doc/manual/src/contributing/cli-guideline.md rename to doc/manual/src/development/cli-guideline.md diff --git a/doc/manual/src/development/contributing.md b/doc/manual/src/development/contributing.md new file mode 100644 index 000000000..7de7489dc --- /dev/null +++ b/doc/manual/src/development/contributing.md @@ -0,0 +1,79 @@ +# Contributing + +## Add a release note + +`doc/manual/rl-next` contains release notes entries for all unreleased changes. + +User-visible changes should come with a release note. + +### Add an entry + +Here's what a complete entry looks like. The file name is not incorporated in the document. + +``` +--- +synopsis: Basically a title +issues: 1234 +prs: 1238 +--- + +Here's one or more paragraphs that describe the change. + +- It's markdown +- Add references to the manual using @docroot@ +``` + +Significant changes should add the following header, which moves them to the top. + +``` +significance: significant +``` + + +See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog). + +### Build process + +Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. + +## Branches + +- [`master`](https://github.com/NixOS/nix/commits/master) + + The main development branch. All changes are approved and merged here. + When developing a change, create a branch based on the latest `master`. + + Maintainers try to [keep it in a release-worthy state](#reverting). + +- [`maintenance-*.*`](https://github.com/NixOS/nix/branches/all?query=maintenance) + + These branches are the subject of backports only, and are + also [kept](#reverting) in a release-worthy state. + + See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) + +- [`latest-release`](https://github.com/NixOS/nix/tree/latest-release) + + The latest patch release of the latest minor version. + + See [`maintainers/release-process.md`](https://github.com/NixOS/nix/blob/master/maintainers/release-process.md) + +- [`backport-*-to-*`](https://github.com/NixOS/nix/branches/all?query=backport) + + Generally branches created by the backport action. + + See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) + +- [_other_](https://github.com/NixOS/nix/branches/all) + + Branches that do not conform to the above patterns should be feature branches. + +## Reverting + +If a change turns out to be merged by mistake, or contain a regression, it may be reverted. +A revert is not a rejection of the contribution, but merely part of an effective development process. +It makes sure that development keeps running smoothly, with minimal uncertainty, and less overhead. +If maintainers have to worry too much about avoiding reverts, they would not be able to merge as much. +By embracing reverts as a good part of the development process, everyone wins. + +However, taking a step back may be frustrating, so maintainers will be extra supportive on the next try. diff --git a/doc/manual/src/contributing/cxx.md b/doc/manual/src/development/cxx.md similarity index 100% rename from doc/manual/src/contributing/cxx.md rename to doc/manual/src/development/cxx.md diff --git a/doc/manual/src/contributing/documentation.md b/doc/manual/src/development/documentation.md similarity index 99% rename from doc/manual/src/contributing/documentation.md rename to doc/manual/src/development/documentation.md index a14ecedd6..63f574ab7 100644 --- a/doc/manual/src/contributing/documentation.md +++ b/doc/manual/src/development/documentation.md @@ -24,7 +24,7 @@ nix build .#^doc and open `./result-doc/share/doc/nix/manual/index.html`. -To build the manual incrementally, [enter the development shell](./hacking.md) and run: +To build the manual incrementally, [enter the development shell](./building.md) and run: ```console make manual-html-open -j $NIX_BUILD_CORES diff --git a/doc/manual/src/contributing/experimental-features.md b/doc/manual/src/development/experimental-features.md similarity index 100% rename from doc/manual/src/contributing/experimental-features.md rename to doc/manual/src/development/experimental-features.md diff --git a/doc/manual/src/contributing/index.md b/doc/manual/src/development/index.md similarity index 77% rename from doc/manual/src/contributing/index.md rename to doc/manual/src/development/index.md index 4d55c17a4..6403c3e66 100644 --- a/doc/manual/src/contributing/index.md +++ b/doc/manual/src/development/index.md @@ -5,4 +5,4 @@ Check the [contributing guide](https://github.com/NixOS/nix/blob/master/CONTRIBU This chapter is a collection of guides for making changes to the code and documentation. -If you're not sure where to start, try to [compile Nix from source](./hacking.md) and consider [making improvements to documentation](./documentation.md). +If you're not sure where to start, try to [compile Nix from source](./building.md) and consider [making improvements to documentation](./documentation.md). diff --git a/doc/manual/src/contributing/json-guideline.md b/doc/manual/src/development/json-guideline.md similarity index 100% rename from doc/manual/src/contributing/json-guideline.md rename to doc/manual/src/development/json-guideline.md diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/development/testing.md similarity index 100% rename from doc/manual/src/contributing/testing.md rename to doc/manual/src/development/testing.md diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index f65ada63a..877c4668b 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -168,7 +168,7 @@ - [impure derivation]{#gloss-impure-derivation} - [An experimental feature](#@docroot@/contributing/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, + [An experimental feature](#@docroot@/development/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, so that they are always rebuilt, and their outputs not reused by subsequent calls to realise them. - [Nix database]{#gloss-nix-database} @@ -353,7 +353,7 @@ Not yet stabilized functionality guarded by named experimental feature flags. These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting. - See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/contributing/experimental-features.md). + See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/development/experimental-features.md). [Nix language]: ./language/index.md diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index 385008d8c..6a168ff3d 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -77,7 +77,7 @@ $ su root # Installing from a binary tarball You can also download a binary tarball that contains Nix and all its dependencies: -- Choose a [version](https://releases.nixos.org/?prefix=nix/) and [system type](../contributing/hacking.md#platforms) +- Choose a [version](https://releases.nixos.org/?prefix=nix/) and [system type](../development/building.md#platforms) - Download and unpack the tarball - Run the installer diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index e916c7360..51b83fc8a 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -113,7 +113,7 @@ Derivations can declare some infrequently used optional attributes. > `nix-build`. If the [`configurable-impure-env` experimental - feature](@docroot@/contributing/experimental-features.md#xp-feature-configurable-impure-env) + feature](@docroot@/development/experimental-features.md#xp-feature-configurable-impure-env) is enabled, these environment variables can also be controlled through the [`impure-env`](@docroot@/command-ref/conf-file.md#conf-impure-env) @@ -226,7 +226,7 @@ Derivations can declare some infrequently used optional attributes. - [`__contentAddressed`]{#adv-attr-__contentAddressed} > **Warning** - > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md). + > This attribute is part of an [experimental feature](@docroot@/development/experimental-features.md). > > To use this attribute, you must enable the > [`ca-derivations`][xp-feature-ca-derivations] experimental feature. @@ -370,6 +370,6 @@ Derivations can declare some infrequently used optional attributes. ensures that the derivation can only be built on a machine with the `kvm` feature. -[xp-feature-ca-derivations]: @docroot@/contributing/experimental-features.md#xp-feature-ca-derivations -[xp-feature-dynamic-derivations]: @docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations -[xp-feature-git-hashing]: @docroot@/contributing/experimental-features.md#xp-feature-git-hashing +[xp-feature-ca-derivations]: @docroot@/development/experimental-features.md#xp-feature-ca-derivations +[xp-feature-dynamic-derivations]: @docroot@/development/experimental-features.md#xp-feature-dynamic-derivations +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/language/constructs/lookup-path.md b/doc/manual/src/language/constructs/lookup-path.md index 11b9fe88c..a2e80280b 100644 --- a/doc/manual/src/language/constructs/lookup-path.md +++ b/doc/manual/src/language/constructs/lookup-path.md @@ -4,11 +4,8 @@ > > *lookup-path* = `<` *identifier* [ `/` *identifier* ]... `>` -A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/types.md#type-path) if the identifier matches a search path entry. - -The value of a lookup path is determined by [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath). - -See [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile) for details on lookup path resolution. +A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/types.md#type-path) if the identifier matches a search path entry in [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath). +The algorithm for lookup path resolution is described in the documentation on [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile). > **Example** > diff --git a/doc/manual/src/language/identifiers.md b/doc/manual/src/language/identifiers.md new file mode 100644 index 000000000..584a2f861 --- /dev/null +++ b/doc/manual/src/language/identifiers.md @@ -0,0 +1,51 @@ +# Identifiers + +An *identifier* is an [ASCII](https://en.wikipedia.org/wiki/ASCII) character sequence that: +- Starts with a letter (`a-z`, `A-Z`) or underscore (`_`) +- Can contain any number of: + - Letters (`a-z`, `A-Z`) + - Digits (`0-9`) + - Underscores (`_`) + - Apostrophes (`'`) + - Hyphens (`-`) +- Is not one of the [keywords](#keywords) + +> **Syntax** +> +> *identifier* ~ `[A-Za-z_][A-Za-z0-9_'-]*` + +# Names + +A *name* can be written as an [identifier](#identifier) or a [string literal](./string-literals.md). + +> **Syntax** +> +> *name* → *identifier* | *string* + +Names are used in [attribute sets](./syntax.md#attrs-literal), [`let` bindings](./syntax.md#let-expressions), and [`inherit`](./syntax.md#inheriting-attributes). +Two names are the same if they represent the same sequence of characters, regardless of whether they are written as identifiers or strings. + +# Keywords + +These keywords are reserved and cannot be used as [identifiers](#identifiers): + +- [`assert`](./syntax.md#assertions) +- [`else`][if] +- [`if`][if] +- [`in`][let] +- [`inherit`](./syntax.md#inheriting-attributes) +- [`let`][let] +- [`or`](./operators.md#attribute-selection) (see note) +- [`rec`](./syntax.md#recursive-sets) +- [`then`][if] +- [`with`](./syntax.md#with-expressions) + +[if]: ./syntax.md#conditionals +[let]: ./syntax.md#let-expressions + +> **Note** +> +> The Nix language evaluator currently allows `or` to be used as a name in some contexts, for backwards compatibility reasons. +> Users are advised not to rely on this. +> +> There are long-standing issues with how `or` is parsed as a name, which can't be resolved without making a breaking change to the language. diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index 9660a764d..27444258a 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -3,7 +3,7 @@ | Name | Syntax | Associativity | Precedence | |----------------------------------------|--------------------------------------------|---------------|------------| | [Attribute selection] | *attrset* `.` *attrpath* \[ `or` *expr* \] | none | 1 | -| Function application | *func* *expr* | left | 2 | +| [Function application] | *func* *expr* | left | 2 | | [Arithmetic negation][arithmetic] | `-` *number* | none | 3 | | [Has attribute] | *attrset* `?` *attrpath* | none | 4 | | List concatenation | *list* `++` *list* | right | 5 | @@ -26,13 +26,17 @@ | Logical conjunction (`AND`) | *bool* `&&` *bool* | left | 12 | | Logical disjunction (`OR`) | *bool* \|\| *bool* | left | 13 | | [Logical implication] | *bool* `->` *bool* | right | 14 | +| [Pipe operator] (experimental) | *expr* `\|>` *func* | left | 15 | +| [Pipe operator] (experimental) | *func* `<\|` *expr* | right | 15 | [string]: ./types.md#type-string [path]: ./types.md#type-path -[number]: ./types.md#type-float -[list]: ./types.md#list +[number]: ./types.md#type-float +[list]: ./types.md#type-list [attribute set]: ./types.md#attribute-set + + ## Attribute selection > **Syntax** @@ -42,13 +46,23 @@ Select the attribute denoted by attribute path *attrpath* from [attribute set] *attrset*. If the attribute doesn’t exist, return the *expr* after `or` if provided, otherwise abort evaluation. -An attribute path is a dot-separated list of [attribute names](./types.md#attribute-set). +[Attribute selection]: #attribute-selection + +## Function application > **Syntax** > -> *attrpath* = *name* [ `.` *name* ]... +> *func* *expr* -[Attribute selection]: #attribute-selection +Apply the callable value *func* to the argument *expr*. Note the absence of any visible operator symbol. +A callable value is either: +- a [user-defined function][function] +- a [built-in][builtins] function +- an attribute set with a [`__functor` attribute](./syntax.md#attr-__functor) + +> **Warning** +> +> [List][list] items are also separated by whitespace, which means that function calls in list items must be enclosed by parentheses. ## Has attribute @@ -69,8 +83,12 @@ After evaluating *attrset* and *attrpath*, the computational complexity is O(log ## Arithmetic -Numbers are type-compatible: -Pure integer operations will always return integers, whereas any operation involving at least one floating point number return a floating point number. +Numbers will retain their type unless mixed with other numeric types: +Pure integer operations will always return integers, whereas any operation involving at least one floating point number returns a floating point number. + +Evaluation of the following numeric operations throws an evaluation error: +- Division by zero +- Integer overflow, that is, any operation yielding a result outside of the representable range of [Nix language integers](./syntax.md#number-literal) See also [Comparison] and [Equality]. @@ -182,3 +200,36 @@ Equivalent to `!`*b1* `||` *b2*. [Logical implication]: #logical-implication +## Pipe operators + +- *a* `|>` *b* is equivalent to *b* *a* +- *a* `<|` *b* is equivalent to *a* *b* + +> **Example** +> +> ``` +> nix-repl> 1 |> builtins.add 2 |> builtins.mul 3 +> 9 +> +> nix-repl> builtins.add 1 <| builtins.mul 2 <| 3 +> 7 +> ``` + +> **Warning** +> +> This syntax is part of an +> [experimental feature](@docroot@/development/experimental-features.md) +> and may change in future releases. +> +> To use this syntax, make sure the +> [`pipe-operators` experimental feature](@docroot@/development/experimental-features.md#xp-feature-pipe-operators) +> is enabled. +> For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): +> +> ``` +> extra-experimental-features = pipe-operators +> ``` + +[Pipe operator]: #pipe-operators +[builtins]: ./builtins.md +[Function application]: #function-application diff --git a/doc/manual/src/language/scope.md b/doc/manual/src/language/scope.md index 5c6aed38d..9373324e2 100644 --- a/doc/manual/src/language/scope.md +++ b/doc/manual/src/language/scope.md @@ -1,14 +1,28 @@ # Scoping rules -Nix is [statically scoped](https://en.wikipedia.org/wiki/Scope_(computer_science)#Lexical_scope), but with multiple scopes and shadowing rules. +A *scope* in the Nix language is a dictionary keyed by [name](./identifiers.md#names), mapping each name to an expression and a *definition type*. +The definition type is either *explicit* or *implicit*. +Each entry in this dictionary is a *definition*. -* primary scope: explicitly-bound variables - * [`let`](./syntax.md#let-expressions) - * [`inherit`](./syntax.md#inheriting-attributes) - * [function](./syntax.md#functions) arguments +Explicit definitions are created by the following expressions: +- [let-expressions](syntax.md#let-expressions) +- [recursive attribute set literals](syntax.md#recursive-sets) (`rec`) +- [function literals](syntax.md#functions) -* secondary scope: implicitly-bound variables - * [`with`](./syntax.md#with-expressions) +Implicit definitions are only created by [with-expressions](./syntax.md#with-expressions). -Primary scope takes precedence over secondary scope. -See [`with`](./syntax.md#with-expressions) for a detailed example. +Every expression is *enclosed* by a scope. +The outermost expression is enclosed by the [built-in, global scope](./builtins.md), which contains only explicit definitions. +The expressions listed above *extend* their enclosing scope by adding new definitions, or replacing existing ones with the same name. +An explicit definition can replace a definition of any type; an implicit definition can only replace another implicit definition. + +Each of the above expressions defines which of its subexpressions are enclosed by the extended scope. +In all other cases, the same scope that encloses an expression is the enclosing scope for its subexpressions. + +The Nix language is [statically scoped](https://en.wikipedia.org/wiki/Scope_(computer_science)#Lexical_scope); +the value of a variable is determined only by the variable's enclosing scope, and not by the dynamic context in which the variable is evaluated. + +> **Note** +> +> Expressions entered into the [Nix REPL](@docroot@/command-ref/new-cli/nix3-repl.md) are enclosed by a scope that can be extended by command line arguments or previous REPL commands. +> These ways of extending scope are not, strictly speaking, part of the Nix language. diff --git a/doc/manual/src/language/string-interpolation.md b/doc/manual/src/language/string-interpolation.md index 1778bdfa0..27780dcbb 100644 --- a/doc/manual/src/language/string-interpolation.md +++ b/doc/manual/src/language/string-interpolation.md @@ -8,6 +8,10 @@ Such a construct is called *interpolated string*, and the expression inside is a [path]: ./types.md#type-path [attribute set]: ./types.md#attribute-set +> **Syntax** +> +> *interpolation_element* → `${` *expression* `}` + ## Examples ### String diff --git a/doc/manual/src/language/string-literals.md b/doc/manual/src/language/string-literals.md new file mode 100644 index 000000000..8f4b75f3e --- /dev/null +++ b/doc/manual/src/language/string-literals.md @@ -0,0 +1,190 @@ +# String literals + +A *string literal* represents a [string](types.md#type-string) value. + +> **Syntax** +> +> *expression* → *string* +> +> *string* → `"` ( *string_char*\* [*interpolation_element*][string interpolation] )* *string_char*\* `"` +> +> *string* → `''` ( *indented_string_char*\* [*interpolation_element*][string interpolation] )* *indented_string_char*\* `''` +> +> *string* → *uri* +> +> *string_char* ~ `[^"$\\]|\$(?!\{)|\\.` +> +> *indented_string_char* ~ `[^$']|\$\$|\$(?!\{)|''[$']|''\\.|'(?!')` +> +> *uri* ~ `[A-Za-z][+\-.0-9A-Za-z]*:[!$%&'*+,\-./0-9:=?@A-Z_a-z~]+` + +Strings can be written in three ways. + +The most common way is to enclose the string between double quotes, e.g., `"foo bar"`. +Strings can span multiple lines. +The results of other expressions can be included into a string by enclosing them in `${ }`, a feature known as [string interpolation]. + +[string interpolation]: ./string-interpolation.md + +The following must be escaped to represent them within a string, by prefixing with a backslash (`\`): + +- Double quote (`"`) + +> **Example** +> +> ```nix +> "\"" +> ``` +> +> "\"" + +- Backslash (`\`) + +> **Example** +> +> ```nix +> "\\" +> ``` +> +> "\\" + +- Dollar sign followed by an opening curly bracket (`${`) – "dollar-curly" + +> **Example** +> +> ```nix +> "\${" +> ``` +> +> "\${" + +The newline, carriage return, and tab characters can be written as `\n`, `\r` and `\t`, respectively. + +A "double-dollar-curly" (`$${`) can be written literally. + +> **Example** +> +> ```nix +> "$${" +> ``` +> +> "$\${" + +String values are output on the terminal with Nix-specific escaping. +Strings written to files will contain the characters encoded by the escaping. + +The second way to write string literals is as an *indented string*, which is enclosed between pairs of *double single-quotes* (`''`), like so: + +```nix +'' +This is the first line. +This is the second line. + This is the third line. +'' +``` + +This kind of string literal intelligently strips indentation from +the start of each line. To be precise, it strips from each line a +number of spaces equal to the minimal indentation of the string as a +whole (disregarding the indentation of empty lines). For instance, +the first and second line are indented two spaces, while the third +line is indented four spaces. Thus, two spaces are stripped from +each line, so the resulting string is + +```nix +"This is the first line.\nThis is the second line.\n This is the third line.\n" +``` + +> **Note** +> +> Whitespace and newline following the opening `''` is ignored if there is no non-whitespace text on the initial line. + +> **Warning** +> +> Prefixed tab characters are not stripped. +> +> > **Example** +> > +> > The following indented string is prefixed with tabs: +> > +> >
''
+> > 	all:
+> > 		@echo hello
+> > ''
+> > 
+> > +> > "\tall:\n\t\t@echo hello\n" + +Indented strings support [string interpolation]. + +The following must be escaped to represent them in an indented string: + +- `$` is escaped by prefixing it with two single quotes (`''`) + +> **Example** +> +> ```nix +> '' +> ''$ +> '' +> ``` +> +> "$\n" + +- `''` is escaped by prefixing it with one single quote (`'`) + +> **Example** +> +> ```nix +> '' +> ''' +> '' +> ``` +> +> "''\n" + +These special characters are escaped as follows: +- Linefeed (`\n`): `''\n` +- Carriage return (`\r`): `''\r` +- Tab (`\t`): `''\t` + +`''\` escapes any other character. + +A "double-dollar-curly" (`$${`) can be written literally. + +> **Example** +> +> ```nix +> '' +> $${ +> '' +> ``` +> +> "$\${\n" + +Indented strings are primarily useful in that they allow multi-line +string literals to follow the indentation of the enclosing Nix +expression, and that less escaping is typically necessary for +strings representing languages such as shell scripts and +configuration files because `''` is much less common than `"`. +Example: + +```nix +stdenv.mkDerivation { +... +postInstall = + '' + mkdir $out/bin $out/etc + cp foo $out/bin + echo "Hello World" > $out/etc/foo.conf + ${if enableBar then "cp bar $out/bin" else ""} + ''; +... +} +``` + +Finally, as a convenience, *URIs* as defined in appendix B of +[RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as +is*, without quotes. For instance, the string +`"http://example.org/foo.tar.bz2"` can also be written as +`http://example.org/foo.tar.bz2`. diff --git a/doc/manual/src/language/syntax.md b/doc/manual/src/language/syntax.md index b0779ea95..506afbea1 100644 --- a/doc/manual/src/language/syntax.md +++ b/doc/manual/src/language/syntax.md @@ -6,175 +6,7 @@ This section covers syntax and semantics of the Nix language. ### String {#string-literal} - *Strings* can be written in three ways. - - The most common way is to enclose the string between double quotes, e.g., `"foo bar"`. - Strings can span multiple lines. - The results of other expressions can be included into a string by enclosing them in `${ }`, a feature known as [string interpolation]. - - [string interpolation]: ./string-interpolation.md - - The following must be escaped to represent them within a string, by prefixing with a backslash (`\`): - - - Double quote (`"`) - - > **Example** - > - > ```nix - > "\"" - > ``` - > - > "\"" - - - Backslash (`\`) - - > **Example** - > - > ```nix - > "\\" - > ``` - > - > "\\" - - - Dollar sign followed by an opening curly bracket (`${`) – "dollar-curly" - - > **Example** - > - > ```nix - > "\${" - > ``` - > - > "\${" - - The newline, carriage return, and tab characters can be written as `\n`, `\r` and `\t`, respectively. - - A "double-dollar-curly" (`$${`) can be written literally. - - > **Example** - > - > ```nix - > "$${" - > ``` - > - > "$\${" - - String values are output on the terminal with Nix-specific escaping. - Strings written to files will contain the characters encoded by the escaping. - - The second way to write string literals is as an *indented string*, which is enclosed between pairs of *double single-quotes* (`''`), like so: - - ```nix - '' - This is the first line. - This is the second line. - This is the third line. - '' - ``` - - This kind of string literal intelligently strips indentation from - the start of each line. To be precise, it strips from each line a - number of spaces equal to the minimal indentation of the string as a - whole (disregarding the indentation of empty lines). For instance, - the first and second line are indented two spaces, while the third - line is indented four spaces. Thus, two spaces are stripped from - each line, so the resulting string is - - ```nix - "This is the first line.\nThis is the second line.\n This is the third line.\n" - ``` - - > **Note** - > - > Whitespace and newline following the opening `''` is ignored if there is no non-whitespace text on the initial line. - - > **Warning** - > - > Prefixed tab characters are not stripped. - > - > > **Example** - > > - > > The following indented string is prefixed with tabs: - > > - > > '' - > > all: - > > @echo hello - > > '' - > > - > > "\tall:\n\t\t@echo hello\n" - - Indented strings support [string interpolation]. - - The following must be escaped to represent them in an indented string: - - - `$` is escaped by prefixing it with two single quotes (`''`) - - > **Example** - > - > ```nix - > '' - > ''$ - > '' - > ``` - > - > "$\n" - - - `''` is escaped by prefixing it with one single quote (`'`) - - > **Example** - > - > ```nix - > '' - > ''' - > '' - > ``` - > - > "''\n" - - These special characters are escaped as follows: - - Linefeed (`\n`): `''\n` - - Carriage return (`\r`): `''\r` - - Tab (`\t`): `''\t` - - `''\` escapes any other character. - - A "double-dollar-curly" (`$${`) can be written literally. - - > **Example** - > - > ```nix - > '' - > $${ - > '' - > ``` - > - > "$\${\n" - - Indented strings are primarily useful in that they allow multi-line - string literals to follow the indentation of the enclosing Nix - expression, and that less escaping is typically necessary for - strings representing languages such as shell scripts and - configuration files because `''` is much less common than `"`. - Example: - - ```nix - stdenv.mkDerivation { - ... - postInstall = - '' - mkdir $out/bin $out/etc - cp foo $out/bin - echo "Hello World" > $out/etc/foo.conf - ${if enableBar then "cp bar $out/bin" else ""} - ''; - ... - } - ``` - - Finally, as a convenience, *URIs* as defined in appendix B of - [RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as - is*, without quotes. For instance, the string - `"http://example.org/foo.tar.bz2"` can also be written as - `http://example.org/foo.tar.bz2`. +See [String literals](string-literals.md). ### Number {#number-literal} @@ -183,6 +15,13 @@ This section covers syntax and semantics of the Nix language. Numbers, which can be *integers* (like `123`) or *floating point* (like `123.43` or `.27e13`). + Integers in the Nix language are 64-bit [two's complement] signed integers, with a range of -9223372036854775808 to 9223372036854775807, inclusive. + + [two's complement]: https://en.wikipedia.org/wiki/Two%27s_complement + + Note that negative numeric literals are actually parsed as unary negation of positive numeric literals. + This means that the minimum integer `-9223372036854775808` cannot be written as-is as a literal, since the positive number `9223372036854775808` is one past the maximum range. + See [arithmetic] and [comparison] operators for semantics. [arithmetic]: ./operators.md#arithmetic @@ -247,37 +86,76 @@ Elements in a list can be accessed using [`builtins.elemAt`](./builtins.md#built ## Attribute Set {#attrs-literal} -An attribute set is a collection of name-value-pairs (called *attributes*) enclosed in curly brackets (`{ }`). +An attribute set is a collection of name-value-pairs called *attributes*. -An attribute name can be an identifier or a [string](#string). -An identifier must start with a letter (`a-z`, `A-Z`) or underscore (`_`), and can otherwise contain letters (`a-z`, `A-Z`), numbers (`0-9`), underscores (`_`), apostrophes (`'`), or dashes (`-`). +Attribute sets are written enclosed in curly brackets (`{ }`). +Attribute names and attribute values are separated by an equal sign (`=`). +Each value can be an arbitrary expression, terminated by a semicolon (`;`) + +An attribute name is a string without context, and is denoted by a [name] (an [identifier](./identifiers.md#identifiers) or [string literal](string-literals.md)). + +[name]: ./identifiers.md#names > **Syntax** > -> *name* = *identifier* | *string* \ -> *identifier* ~ `[a-zA-Z_][a-zA-Z0-9_'-]*` - -Names and values are separated by an equal sign (`=`). -Each value is an arbitrary expression terminated by a semicolon (`;`). - -> **Syntax** -> -> *attrset* = `{` [ *name* `=` *expr* `;` ]... `}` +> *attrset* → `{` { *name* `=` *expr* `;` } `}` Attributes can appear in any order. -An attribute name may only occur once. +An attribute name may only occur once in each attribute set. -Example: +> **Example** +> +> This defines an attribute set with attributes named: +> - `x` with the value `123`, an integer +> - `text` with the value `"Hello"`, a string +> - `y` where the value is the result of applying the function `f` to the attribute set `{ bla = 456; }` +> +> ```nix +> { +> x = 123; +> text = "Hello"; +> y = f { bla = 456; }; +> } +> ``` -```nix -{ - x = 123; - text = "Hello"; - y = f { bla = 456; }; -} -``` +Attributes in nested attribute sets can be written using *attribute paths*. -This defines a set with attributes named `x`, `text`, `y`. +> **Syntax** +> +> *attrset* → `{` { *attrpath* `=` *expr* `;` } `}` + +An attribute path is a dot-separated list of [names][name]. + +> **Syntax** +> +> *attrpath* = *name* { `.` *name* } + + + +> **Example** +> +> ```nix +> { a.b.c = 1; a.b.d = 2; } +> ``` +> +> { +> a = { +> b = { +> c = 1; +> d = 2; +> }; +> }; +> } + +Attribute names can also be set implicitly by using the [`inherit` keyword](#inheriting-attributes). + +> **Example** +> +> ```nix +> { inherit (builtins) true; } +> ``` +> +> { true = true; } Attributes can be accessed with the [`.` operator](./operators.md#attribute-selection). @@ -340,7 +218,7 @@ a string), that attribute is simply not added to the set: This will evaluate to `{}` if `foo` evaluates to `false`. -A set that has a `__functor` attribute whose value is callable (i.e. is +A set that has a [`__functor`]{#attr-__functor} attribute whose value is callable (i.e. is itself a function or a set with a `__functor` attribute whose value is callable) can be applied as if it were a function, with the set itself passed in first , e.g., diff --git a/doc/manual/src/language/types.md b/doc/manual/src/language/types.md index 229756e6b..82184a8b0 100644 --- a/doc/manual/src/language/types.md +++ b/doc/manual/src/language/types.md @@ -45,7 +45,7 @@ The function [`builtins.isBool`](builtins.md#builtins-isBool) can be used to det A _string_ in the Nix language is an immutable, finite-length sequence of bytes, along with a [string context](string-context.md). Nix does not assume or support working natively with character encodings. -String values without string context can be expressed as [string literals](syntax.md#string-literal). +String values without string context can be expressed as [string literals](string-literals.md). The function [`builtins.isString`](builtins.md#builtins-isString) can be used to determine if a value is a string. ### Path {#type-path} diff --git a/doc/manual/src/language/variables.md b/doc/manual/src/language/variables.md new file mode 100644 index 000000000..af6aff8a2 --- /dev/null +++ b/doc/manual/src/language/variables.md @@ -0,0 +1,10 @@ +# Variables + +A *variable* is an [identifier](identifiers.md) used as an expression. + +> **Syntax** +> +> *expression* → *identifier* + +A variable must have the same name as a definition in the [scope](./scope.md) that encloses it. +The value of a variable is the value of the corresponding expression in the enclosing scope. diff --git a/doc/manual/src/protocols/derivation-aterm.md b/doc/manual/src/protocols/derivation-aterm.md index e58b602a3..1ba757ae0 100644 --- a/doc/manual/src/protocols/derivation-aterm.md +++ b/doc/manual/src/protocols/derivation-aterm.md @@ -14,6 +14,6 @@ Derivations are serialised in one of the following formats: DrvWithVersion(, ...) ``` - The only `version-string`s that are in use today are for [experimental features](@docroot@/contributing/experimental-features.md): + The only `version-string`s that are in use today are for [experimental features](@docroot@/development/experimental-features.md): - - `"xp-dyn-drv"` for the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. + - `"xp-dyn-drv"` for the [`dynamic-derivations`](@docroot@/development/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. diff --git a/doc/manual/src/protocols/json/derivation.md b/doc/manual/src/protocols/json/derivation.md index f881dd703..2f85340d6 100644 --- a/doc/manual/src/protocols/json/derivation.md +++ b/doc/manual/src/protocols/json/derivation.md @@ -3,7 +3,7 @@ > **Warning** > > This JSON format is currently -> [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) +> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) > and subject to change. The JSON serialization of a diff --git a/doc/manual/src/protocols/json/store-object-info.md b/doc/manual/src/protocols/json/store-object-info.md index 9f647a96c..6b4f48437 100644 --- a/doc/manual/src/protocols/json/store-object-info.md +++ b/doc/manual/src/protocols/json/store-object-info.md @@ -3,7 +3,7 @@ > **Warning** > > This JSON format is currently -> [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) +> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) > and subject to change. Info about a [store object]. diff --git a/doc/manual/src/release-notes/rl-2.18.md b/doc/manual/src/release-notes/rl-2.18.md index 4bbc52b50..eb26fc9e7 100644 --- a/doc/manual/src/release-notes/rl-2.18.md +++ b/doc/manual/src/release-notes/rl-2.18.md @@ -13,7 +13,7 @@ - The `discard-references` feature has been stabilized. This means that the - [unsafeDiscardReferences](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) + [unsafeDiscardReferences](@docroot@/development/experimental-features.md#xp-feature-discard-references) attribute is no longer guarded by an experimental flag and can be used freely. @@ -21,7 +21,7 @@ This only affects `nix-build --json` when "building" non-derivation things like fetched sources, which is a no-op. - A new builtin [`outputOf`](@docroot@/language/builtins.md#builtins-outputOf) has been added. - It is part of the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. + It is part of the [`dynamic-derivations`](@docroot@/development/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. - Flake follow paths at depths greater than 2 are now handled correctly, preventing "follows a non-existent input" errors. diff --git a/doc/manual/src/release-notes/rl-2.19.md b/doc/manual/src/release-notes/rl-2.19.md index ba6eb9c64..e2e2f85cc 100644 --- a/doc/manual/src/release-notes/rl-2.19.md +++ b/doc/manual/src/release-notes/rl-2.19.md @@ -17,8 +17,8 @@ - `nix-shell` shebang lines now support single-quoted arguments. -- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). - This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). +- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/development/experimental-features.md#xp-fetch-tree). + This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/development/experimental-features.md#xp-fetch-tree). - The interface for creating and updating lock files has been overhauled: @@ -33,7 +33,7 @@ - The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables. They are superceded by `nix flake update`. -- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). +- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). - [`nix path-info --json`](@docroot@/command-ref/new-cli/nix3-path-info.md) (experimental) now returns a JSON map rather than JSON list. diff --git a/doc/manual/src/release-notes/rl-2.23.md b/doc/manual/src/release-notes/rl-2.23.md index 3c59b8583..ac842fdc0 100644 --- a/doc/manual/src/release-notes/rl-2.23.md +++ b/doc/manual/src/release-notes/rl-2.23.md @@ -14,7 +14,7 @@ - Modify `nix derivation {add,show}` JSON format [#9866](https://github.com/NixOS/nix/issues/9866) [#10722](https://github.com/NixOS/nix/pull/10722) - The JSON format for derivations has been slightly revised to better conform to our [JSON guidelines](@docroot@/contributing/cli-guideline.md#returning-future-proof-json). + The JSON format for derivations has been slightly revised to better conform to our [JSON guidelines](@docroot@/development/cli-guideline.md#returning-future-proof-json). In particular, the hash algorithm and content addressing method of content-addresed derivation outputs are now separated into two fields `hashAlgo` and `method`, rather than one field with an arcane `:`-separated format. @@ -89,7 +89,7 @@ This makes records of this sort more self-describing, and easier to consume programmatically. We will follow this design principle going forward; - the [JSON guidelines](@docroot@/contributing/json-guideline.md) in the contributing section have been updated accordingly. + the [JSON guidelines](@docroot@/development/json-guideline.md) in the contributing section have been updated accordingly. - Large path warnings [#10661](https://github.com/NixOS/nix/pull/10661) diff --git a/doc/manual/src/release-notes/rl-2.24.md b/doc/manual/src/release-notes/rl-2.24.md new file mode 100644 index 000000000..5bcc1d79c --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.24.md @@ -0,0 +1,324 @@ +# Release 2.24.0 (2024-07-31) + +### Significant changes + +- Harden user sandboxing + + The build directory has been hardened against interference with the outside world by nesting it inside another directory owned by (and only readable by) the daemon user. + + This is a low severity security fix, [CVE-2024-38531](https://www.cve.org/CVERecord?id=CVE-2024-38531). + + Credit: [**@alois31**](https://github.com/alois31), [**Linus Heckemann (@lheckemann)**](https://github.com/lheckemann) + Co-authors: [**@edolstra**](https://github.com/edolstra) + +- `nix-shell ` looks for `shell.nix` [#496](https://github.com/NixOS/nix/issues/496) [#2279](https://github.com/NixOS/nix/issues/2279) [#4529](https://github.com/NixOS/nix/issues/4529) [#5431](https://github.com/NixOS/nix/issues/5431) [#11053](https://github.com/NixOS/nix/issues/11053) [#11057](https://github.com/NixOS/nix/pull/11057) + + `nix-shell $x` now looks for `$x/shell.nix` when `$x` resolves to a directory. + + Although this might be seen as a breaking change, its primarily interactive usage makes it a minor issue. + This adjustment addresses a commonly reported problem. + + This also applies to `nix-shell` shebang scripts. Consider the following example: + + ```shell + #!/usr/bin/env nix-shell + #!nix-shell -i bash + ``` + + This will now load `shell.nix` from the script's directory, if it exists; `default.nix` otherwise. + + The old behavior can be opted into by setting the option [`nix-shell-always-looks-for-shell-nix`](@docroot@/command-ref/conf-file.md#conf-nix-shell-always-looks-for-shell-nix) to `false`. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + +- `nix-repl`'s `:doc` shows documentation comments [#3904](https://github.com/NixOS/nix/issues/3904) [#10771](https://github.com/NixOS/nix/issues/10771) [#1652](https://github.com/NixOS/nix/pull/1652) [#9054](https://github.com/NixOS/nix/pull/9054) [#11072](https://github.com/NixOS/nix/pull/11072) + + `nix repl` has a `:doc` command that previously only rendered documentation for internally defined functions. + This feature has been extended to also render function documentation comments, in accordance with [RFC 145]. + + Example: + + ``` + nix-repl> :doc lib.toFunction + Function toFunction + … defined at /home/user/h/nixpkgs/lib/trivial.nix:1072:5 + + Turns any non-callable values into constant functions. Returns + callable values as is. + + Inputs + + v + + : Any value + + Examples + + :::{.example} + + ## lib.trivial.toFunction usage example + + | nix-repl> lib.toFunction 1 2 + | 1 + | + | nix-repl> lib.toFunction (x: x + 1) 2 + | 3 + + ::: + ``` + + Known limitations: + - It does not render documentation for "formals", such as `{ /** the value to return */ x, ... }: x`. + - Some extensions to markdown are not yet supported, as you can see in the example above. + + We'd like to acknowledge [Yingchi Long (@inclyc)](https://github.com/inclyc) for proposing a proof of concept for this functionality in [#9054](https://github.com/NixOS/nix/pull/9054), as well as [@sternenseemann](https://github.com/sternenseemann) and [Johannes Kirschbauer (@hsjobeki)](https://github.com/hsjobeki) for their contributions, proposals, and their work on [RFC 145]. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + + [RFC 145]: https://github.com/NixOS/rfcs/pull/145 + +### Other changes + +- Solve `cached failure of attribute X` [#9165](https://github.com/NixOS/nix/issues/9165) [#10513](https://github.com/NixOS/nix/issues/10513) [#10564](https://github.com/NixOS/nix/pull/10564) + + This eliminates all "cached failure of attribute X" messages by forcing evaluation of the original value when needed to show the exception to the user. This enhancement improves error reporting by providing the underlying message and stack trace. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Run the flake regressions test suite [#10603](https://github.com/NixOS/nix/pull/10603) + + This update introduces a GitHub action to run a subset of the [flake regressions test suite](https://github.com/NixOS/flake-regressions), which includes 259 flakes with their expected evaluation results. Currently, the action runs the first 25 flakes due to the full test suite's extensive runtime. A manually triggered action may be implemented later to run the entire test suite. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Support unit prefixes in configuration settings [#10668](https://github.com/NixOS/nix/pull/10668) + + Configuration settings in Nix now support unit prefixes, allowing for more intuitive and readable configurations. For example, you can now specify [`--min-free 1G`](@docroot@/command-ref/opt-common.md#opt-min-free) to set the minimum free space to 1 gigabyte. + + This enhancement was extracted from [#7851](https://github.com/NixOS/nix/pull/7851) and is also useful for PR [#10661](https://github.com/NixOS/nix/pull/10661). + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- `nix build`: show all FOD errors with `--keep-going` [#10734](https://github.com/NixOS/nix/pull/10734) + + The [`nix build`](@docroot@/command-ref/new-cli/nix3-build.md) command has been updated to improve the behavior of the [`--keep-going`] flag. Now, when `--keep-going` is used, all hash-mismatch errors of failing fixed-output derivations (FODs) are displayed, similar to the behavior for other build failures. This enhancement ensures that all relevant build errors are shown, making it easier for users to update multiple derivations at once or to diagnose and fix issues. + + Author: [**Jörg Thalheim (@Mic92)**](https://github.com/Mic92), [**Maximilian Bosch (@Ma27)**](https://github.com/Ma27) + + [`--keep-going`](@docroot@/command-ref/opt-common.md#opt-keep-going) + +- Build with Meson [#2503](https://github.com/NixOS/nix/issues/2503) [#10378](https://github.com/NixOS/nix/pull/10378) [#10855](https://github.com/NixOS/nix/pull/10855) [#10904](https://github.com/NixOS/nix/pull/10904) [#10908](https://github.com/NixOS/nix/pull/10908) [#10914](https://github.com/NixOS/nix/pull/10914) [#10933](https://github.com/NixOS/nix/pull/10933) [#10936](https://github.com/NixOS/nix/pull/10936) [#10954](https://github.com/NixOS/nix/pull/10954) [#10955](https://github.com/NixOS/nix/pull/10955) [#10963](https://github.com/NixOS/nix/pull/10963) [#10967](https://github.com/NixOS/nix/pull/10967) [#10973](https://github.com/NixOS/nix/pull/10973) [#11034](https://github.com/NixOS/nix/pull/11034) [#11054](https://github.com/NixOS/nix/pull/11054) [#11055](https://github.com/NixOS/nix/pull/11055) [#11060](https://github.com/NixOS/nix/pull/11060) [#11064](https://github.com/NixOS/nix/pull/11064) [#11155](https://github.com/NixOS/nix/pull/11155) + + These changes aim to replace the use of autotools and `make` with Meson for building various components of Nix. Additionally, each library is built in its own derivation, leveraging Meson's "subprojects" feature to allow a single development shell for building all libraries while also supporting separate builds. This approach aims to improve productivity and build modularity, compared to both make and a monolithic Meson-based derivation. + + Special thanks to everyone who has contributed to the Meson port, particularly [**@p01arst0rm**](https://github.com/p01arst0rm) and [**@Qyriad**](https://github.com/Qyriad). + + Authors: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314), [**Tom Bereknyei**](https://github.com/tomberek), [**Théophane Hufschmitt (@thufschmitt)**](https://github.com/thufschmitt), [**Valentin Gagarin (@fricklerhandwerk)**](https://github.com/fricklerhandwerk), [**Robert Hensing (@roberth)**](https://github.com/roberth) + Co-authors: [**@p01arst0rm**](https://github.com/p01arst0rm), [**@Qyriad**](https://github.com/Qyriad) + +- Evaluation cache: fix cache regressions [#10570](https://github.com/NixOS/nix/issues/10570) [#11086](https://github.com/NixOS/nix/pull/11086) + + This update addresses two bugs in the evaluation cache system: + + 1. Regression in #10570: The evaluation cache was not being persisted in `nix develop`. + 2. Nix could sometimes try to commit the evaluation cache SQLite transaction without there being an active transaction, resulting in non-error errors being printed. + + Author: [**Lexi Mattick (@kognise)**](https://github.com/kognise) + +- Introduce `libnixflake` [#9063](https://github.com/NixOS/nix/pull/9063) + + A new library, `libnixflake`, has been introduced to better separate the Flakes layer within Nix. This change refactors the codebase to encapsulate Flakes-specific functionality within its own library. + + See the commits in the pull request for detailed changes, with the only significant code modifications happening in the initial commit. + + This change was alluded to in [RFC 134](https://github.com/nixos/rfcs/blob/master/rfcs/0134-nix-store-layer.md) and is a step towards a more modular and maintainable codebase. + + Author: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314) + +- CLI options `--arg-from-file` and `--arg-from-stdin` [#9913](https://github.com/NixOS/nix/pull/9913) + +- The `--debugger` now prints source location information, instead of the + pointers of source location information. Before: + + ``` + nix-repl> :bt + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + 0x600001522598 + ``` + + After: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in + ``` + +- Stop vendoring `toml11` + + We don't apply any patches to it, and vendoring it locks users into + bugs (it hasn't been updated since its introduction in late 2021). + + Author: [**Winter (@winterqt)**](https://github.com/winterqt) + +- Rename hash format `base32` to `nix32` [#8678](https://github.com/NixOS/nix/pull/8678) + + Hash format `base32` was renamed to `nix32` since it used a special nix-specific character set for + [Base32](https://en.wikipedia.org/wiki/Base32). + + **Deprecation**: Use `nix32` instead of `base32` as `toHashFormat` + + For the builtin `convertHash`, the `toHashFormat` parameter now accepts the same hash formats as the `--to`/`--from` + parameters of the `nix hash conert` command: `"base16"`, `"nix32"`, `"base64"`, and `"sri"`. The former `"base32"` value + remains as a deprecated alias for `"nix32"`. Please convert your code from: + + ```nix + builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base32";} + ``` + + to + + ```nix + builtins.convertHash { inherit hash hashAlgo; toHashFormat = "nix32";} + ``` + +- Add `pipe-operators` experimental feature [#11131](https://github.com/NixOS/nix/pull/11131) + + This is a draft implementation of [RFC 0148](https://github.com/NixOS/rfcs/pull/148). + + The `pipe-operators` experimental feature adds [`<|` and `|>` operators][pipe operators] to the Nix language. + *a* `|>` *b* is equivalent to the function application *b* *a*, and + *a* `<|` *b* is equivalent to the function application *a* *b*. + + For example: + + ``` + nix-repl> 1 |> builtins.add 2 |> builtins.mul 3 + 9 + + nix-repl> builtins.add 1 <| builtins.mul 2 <| 3 + 7 + ``` + + `<|` and `|>` are right and left associative, respectively, and have lower precedence than any other operator. + These properties may change in future releases. + + See [the RFC](https://github.com/NixOS/rfcs/pull/148) for more examples and rationale. + + [pipe operators]: @docroot@/language/operators.md#pipe-operators + +- `nix-shell` shebang uses relative path [#4232](https://github.com/NixOS/nix/issues/4232) [#5088](https://github.com/NixOS/nix/pull/5088) [#11058](https://github.com/NixOS/nix/pull/11058) + + + Relative [path](@docroot@/language/types.md#type-path) literals in `nix-shell` shebang scripts' options are now resolved relative to the [script's location](@docroot@/glossary.md?highlight=base%20directory#gloss-base-directory). + Previously they were resolved relative to the current working directory. + + For example, consider the following script in `~/myproject/say-hi`: + + ```shell + #!/usr/bin/env nix-shell + #!nix-shell --expr 'import ./shell.nix' + #!nix-shell --arg toolset './greeting-tools.nix' + #!nix-shell -i bash + hello + ``` + + Older versions of `nix-shell` would resolve `shell.nix` relative to the current working directory, such as the user's home directory in this example: + + ```console + [hostname:~]$ ./myproject/say-hi + error: + … while calling the 'import' builtin + at «string»:1:2: + 1| (import ./shell.nix) + | ^ + + error: path '/home/user/shell.nix' does not exist + ``` + + Since this release, `nix-shell` resolves `shell.nix` relative to the script's location, and `~/myproject/shell.nix` is used. + + ```console + $ ./myproject/say-hi + Hello, world! + ``` + + **Opt-out** + + This is technically a breaking change, so we have added an option so you can adapt independently of your Nix update. + The old behavior can be opted into by setting the option [`nix-shell-shebang-arguments-relative-to-script`](@docroot@/command-ref/conf-file.md#conf-nix-shell-shebang-arguments-relative-to-script) to `false`. + This option will be removed in a future release. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + +- Improve handling of tarballs that don't consist of a single top-level directory [#11195](https://github.com/NixOS/nix/pull/11195) + + In previous Nix releases, the tarball fetcher (used by `builtins.fetchTarball`) erroneously merged top-level directories into a single directory, and silently discarded top-level files that are not directories. This is no longer the case. The new behaviour is that *only* if the tarball consists of a single directory, the top-level path component of the files in the tarball is removed (similar to `tar`'s `--strip-components=1`). + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Improve handling of tarballs that don't consist of a single top-level directory [#11195](https://github.com/NixOS/nix/pull/11195) + + In previous Nix releases, the tarball fetcher (used by `builtins.fetchTarball`) erroneously merged top-level directories into a single directory, and silently discarded top-level files that are not directories. This is no longer the case. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Setting to warn about large paths [#10778](https://github.com/NixOS/nix/pull/10778) + + Nix can now warn when evaluation of a Nix expression causes a large + path to be copied to the Nix store. The threshold for this warning can + be configured using the `warn-large-path-threshold` setting, + e.g. `--warn-large-path-threshold 100M`. + + +# Contributors + +This release was made possible by the following 43 contributors: + +- Andreas Rammhold [**(@andir)**](https://github.com/andir) +- Andrew Marshall [**(@amarshall)**](https://github.com/amarshall) +- Brian McKenna [**(@puffnfresh)**](https://github.com/puffnfresh) +- Cameron [**(@SkamDart)**](https://github.com/SkamDart) +- Cole Helbling [**(@cole-h)**](https://github.com/cole-h) +- Corbin Simpson [**(@MostAwesomeDude)**](https://github.com/MostAwesomeDude) +- Eelco Dolstra [**(@edolstra)**](https://github.com/edolstra) +- Emily [**(@emilazy)**](https://github.com/emilazy) +- Enno Richter [**(@elohmeier)**](https://github.com/elohmeier) +- Farid Zakaria [**(@fzakaria)**](https://github.com/fzakaria) +- HaeNoe [**(@haenoe)**](https://github.com/haenoe) +- Hamir Mahal [**(@hamirmahal)**](https://github.com/hamirmahal) +- Harmen [**(@alicebob)**](https://github.com/alicebob) +- Ivan Trubach [**(@tie)**](https://github.com/tie) +- Jared Baur [**(@jmbaur)**](https://github.com/jmbaur) +- John Ericson [**(@Ericson2314)**](https://github.com/Ericson2314) +- Jonathan De Troye [**(@detroyejr)**](https://github.com/detroyejr) +- Jörg Thalheim [**(@Mic92)**](https://github.com/Mic92) +- Klemens Nanni [**(@klemensn)**](https://github.com/klemensn) +- Las Safin [**(@L-as)**](https://github.com/L-as) +- Lexi Mattick [**(@kognise)**](https://github.com/kognise) +- Matthew Bauer [**(@matthewbauer)**](https://github.com/matthewbauer) +- Max “Goldstein” Siling [**(@GoldsteinE)**](https://github.com/GoldsteinE) +- Mingye Wang [**(@Artoria2e5)**](https://github.com/Artoria2e5) +- Philip Taron [**(@philiptaron)**](https://github.com/philiptaron) +- Pierre Bourdon [**(@delroth)**](https://github.com/delroth) +- Pino Toscano [**(@pinotree)**](https://github.com/pinotree) +- RTUnreal [**(@RTUnreal)**](https://github.com/RTUnreal) +- Robert Hensing [**(@roberth)**](https://github.com/roberth) +- Romain Neil [**(@romain-neil)**](https://github.com/romain-neil) +- Ryan Hendrickson [**(@rhendric)**](https://github.com/rhendric) +- Sergei Trofimovich [**(@trofi)**](https://github.com/trofi) +- Shogo Takata [**(@pineapplehunter)**](https://github.com/pineapplehunter) +- Siddhant Kumar [**(@siddhantk232)**](https://github.com/siddhantk232) +- Silvan Mosberger [**(@infinisil)**](https://github.com/infinisil) +- Théophane Hufschmitt [**(@thufschmitt)**](https://github.com/thufschmitt) +- Valentin Gagarin [**(@fricklerhandwerk)**](https://github.com/fricklerhandwerk) +- Winter [**(@winterqt)**](https://github.com/winterqt) +- jade [**(@lf-)**](https://github.com/lf-) +- kirillrdy [**(@kirillrdy)**](https://github.com/kirillrdy) +- pennae [**(@pennae)**](https://github.com/pennae) +- poweredbypie [**(@poweredbypie)**](https://github.com/poweredbypie) +- tomberek [**(@tomberek)**](https://github.com/tomberek) diff --git a/doc/manual/src/release-notes/rl-2.4.md b/doc/manual/src/release-notes/rl-2.4.md index 8b566fc7b..1201e53b6 100644 --- a/doc/manual/src/release-notes/rl-2.4.md +++ b/doc/manual/src/release-notes/rl-2.4.md @@ -23,7 +23,7 @@ more than 2800 commits from 195 contributors since release 2.3. * The **`nix` command** has seen a lot of work and is now almost at feature parity with the old command-line interface (the `nix-*` commands). It aims to be [more modern, consistent and pleasant to - use](../contributing/cli-guideline.md) than the old CLI. It is still + use](../development/cli-guideline.md) than the old CLI. It is still marked as experimental but its interface should not change much anymore in future releases. diff --git a/doc/manual/src/store/file-system-object/content-address.md b/doc/manual/src/store/file-system-object/content-address.md index 1c63c52eb..410d7fb7c 100644 --- a/doc/manual/src/store/file-system-object/content-address.md +++ b/doc/manual/src/store/file-system-object/content-address.md @@ -82,4 +82,4 @@ In the future, we may support a Git-like hash for such file system objects, or w [file system object]: ../file-system-object.md [store object]: ../store-object.md -[xp-feature-git-hashing]: @docroot@/contributing/experimental-features.md#xp-feature-git-hashing +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/store/store-object/content-address.md b/doc/manual/src/store/store-object/content-address.md index f6f982035..02dce2836 100644 --- a/doc/manual/src/store/store-object/content-address.md +++ b/doc/manual/src/store/store-object/content-address.md @@ -92,4 +92,4 @@ becomes more widespread, this restriction will be revisited. [fso-ca]: ../file-system-object/content-address.md [sp-spec]: @docroot@/protocols/store-path.md -[xp-feature-git-hashing]: @docroot@/contributing/experimental-features.md#xp-feature-git-hashing +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/flake.lock b/flake.lock index 1d59439da..b5d0b881c 100644 --- a/flake.lock +++ b/flake.lock @@ -3,11 +3,11 @@ "flake-compat": { "flake": false, "locked": { - "lastModified": 1673956053, - "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", "owner": "edolstra", "repo": "flake-compat", - "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", "type": "github" }, "original": { @@ -23,11 +23,11 @@ ] }, "locked": { - "lastModified": 1712014858, - "narHash": "sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm+GpZNw=", + "lastModified": 1719994518, + "narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "9126214d0a59633752a136528f5f3b9aa8565b7d", + "rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7", "type": "github" }, "original": { @@ -36,49 +36,60 @@ "type": "github" } }, - "flake-utils": { + "git-hooks-nix": { + "inputs": { + "flake-compat": [], + "gitignore": [], + "nixpkgs": [ + "nixpkgs" + ], + "nixpkgs-stable": [ + "nixpkgs" + ] + }, "locked": { - "lastModified": 1667395993, - "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f", + "lastModified": 1721042469, + "narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=", + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "f451c19376071a90d8c58ab1a953c6e9840527fd", "type": "github" }, "original": { - "owner": "numtide", - "repo": "flake-utils", + "owner": "cachix", + "repo": "git-hooks.nix", "type": "github" } }, "libgit2": { "flake": false, "locked": { - "lastModified": 1697646580, - "narHash": "sha256-oX4Z3S9WtJlwvj0uH9HlYcWv+x1hqp8mhXl7HsLu2f0=", + "lastModified": 1715853528, + "narHash": "sha256-J2rCxTecyLbbDdsyBWn9w7r3pbKRMkI9E7RvRgAqBdY=", "owner": "libgit2", "repo": "libgit2", - "rev": "45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5", + "rev": "36f7e21ad757a3dacc58cf7944329da6bc1d6e96", "type": "github" }, "original": { "owner": "libgit2", + "ref": "v1.8.1", "repo": "libgit2", "type": "github" } }, "nixpkgs": { "locked": { - "lastModified": 1721560568, - "narHash": "sha256-L61BXz7n/yNzOeZ3FqlnUmxj4145JOVeq9fvQTQzbNM=", + "lastModified": 1723688146, + "narHash": "sha256-sqLwJcHYeWLOeP/XoLwAtYjr01TISlkOfz+NG82pbdg=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "be3ca229c85e978880babdeda9748b14e6aa008f", + "rev": "c3d4ac725177c030b1e289015989da2ad9d56af0", "type": "github" }, "original": { "owner": "NixOS", - "ref": "release-24.05", + "ref": "nixos-24.05", "repo": "nixpkgs", "type": "github" } @@ -115,41 +126,15 @@ "type": "github" } }, - "pre-commit-hooks": { - "inputs": { - "flake-compat": [], - "flake-utils": "flake-utils", - "gitignore": [], - "nixpkgs": [ - "nixpkgs" - ], - "nixpkgs-stable": [ - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1712897695, - "narHash": "sha256-nMirxrGteNAl9sWiOhoN5tIHyjBbVi5e2tgZUgZlK3Y=", - "owner": "cachix", - "repo": "pre-commit-hooks.nix", - "rev": "40e6053ecb65fcbf12863338a6dcefb3f55f1bf8", - "type": "github" - }, - "original": { - "owner": "cachix", - "repo": "pre-commit-hooks.nix", - "type": "github" - } - }, "root": { "inputs": { "flake-compat": "flake-compat", "flake-parts": "flake-parts", + "git-hooks-nix": "git-hooks-nix", "libgit2": "libgit2", "nixpkgs": "nixpkgs", "nixpkgs-23-11": "nixpkgs-23-11", - "nixpkgs-regression": "nixpkgs-regression", - "pre-commit-hooks": "pre-commit-hooks" + "nixpkgs-regression": "nixpkgs-regression" } } }, diff --git a/flake.nix b/flake.nix index a8b550db1..a154ceb7b 100644 --- a/flake.nix +++ b/flake.nix @@ -6,24 +6,22 @@ description = "The purely functional package manager - but super!"; - # TODO switch to nixos-23.11-small - # https://nixpk.gs/pr-tracker.html?pr=291954 - inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-24.05"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; - inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; }; + inputs.libgit2 = { url = "github:libgit2/libgit2/v1.8.1"; flake = false; }; # dev tooling inputs.flake-parts.url = "github:hercules-ci/flake-parts"; - inputs.pre-commit-hooks.url = "github:cachix/pre-commit-hooks.nix"; + inputs.git-hooks-nix.url = "github:cachix/git-hooks.nix"; # work around https://github.com/NixOS/nix/issues/7730 inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; - inputs.pre-commit-hooks.inputs.nixpkgs.follows = "nixpkgs"; - inputs.pre-commit-hooks.inputs.nixpkgs-stable.follows = "nixpkgs"; + inputs.git-hooks-nix.inputs.nixpkgs.follows = "nixpkgs"; + inputs.git-hooks-nix.inputs.nixpkgs-stable.follows = "nixpkgs"; # work around 7730 and https://github.com/NixOS/nix/issues/7807 - inputs.pre-commit-hooks.inputs.flake-compat.follows = ""; - inputs.pre-commit-hooks.inputs.gitignore.follows = ""; + inputs.git-hooks-nix.inputs.flake-compat.follows = ""; + inputs.git-hooks-nix.inputs.gitignore.follows = ""; outputs = inputs@{ self, nixpkgs, nixpkgs-regression, libgit2, ... }: @@ -33,12 +31,6 @@ officialRelease = false; - version = lib.fileContents ./.version + versionSuffix; - versionSuffix = - if officialRelease - then "" - else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}"; - linux32BitSystems = [ "i686-linux" ]; linux64BitSystems = [ "x86_64-linux" "aarch64-linux" ]; linuxSystems = linux32BitSystems ++ linux64BitSystems; @@ -137,21 +129,21 @@ # without "polluting" the top level "`pkgs`" attrset. # This also has the benefit of providing us with a distinct set of packages # we can iterate over. - nixComponents = lib.makeScope final.nixDependencies.newScope (import ./packaging/components.nix); + nixComponents = lib.makeScope final.nixDependencies.newScope (import ./packaging/components.nix { + inherit (final) lib; + inherit officialRelease; + src = self; + }); # The dependencies are in their own scope, so that they don't have to be # in Nixpkgs top level `pkgs` or `nixComponents`. nixDependencies = lib.makeScope final.newScope (import ./packaging/dependencies.nix { - inherit inputs stdenv versionSuffix; + inherit inputs stdenv; pkgs = final; }); nix = final.nixComponents.nix; - nix_noTests = final.nix.override { - doInstallCheck = false; - }; - # See https://github.com/NixOS/nixpkgs/pull/214409 # Remove when fixed in this flake's nixpkgs pre-commit = @@ -176,6 +168,7 @@ linux64BitSystems nixpkgsFor self + officialRelease ; }; @@ -217,6 +210,9 @@ "${nixpkgsPrefix}${pkgName}-${testName}" = test; }) ) + // lib.optionalAttrs (nixpkgs.stdenv.hostPlatform == nixpkgs.stdenv.buildPlatform) { + "${nixpkgsPrefix}nix-functional-tests" = nixpkgs.nixComponents.nix-functional-tests; + } ) // devFlake.checks.${system} or {} ); @@ -226,7 +222,7 @@ # for which we don't apply the full build matrix such as cross or static. inherit (nixpkgsFor.${system}.native) changelog-d; - default = self.packages.${system}.nix; + default = self.packages.${system}.nix-ng; nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-internal-api-docs; nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-external-api-docs; } @@ -234,22 +230,48 @@ // flatMapAttrs { # Components we'll iterate over in the upcoming lambda "nix" = { }; - # Temporarily disabled because GitHub Actions OOM issues. Once - # the old build system is gone and we are back to one build - # system, we should reenable these. - #"nix-util" = { }; - #"nix-store" = { }; - #"nix-fetchers" = { }; + "nix-util" = { }; + "nix-util-c" = { }; + "nix-util-test-support" = { }; + "nix-util-tests" = { }; + + "nix-store" = { }; + "nix-store-c" = { }; + "nix-store-test-support" = { }; + "nix-store-tests" = { }; + + "nix-fetchers" = { }; + "nix-fetchers-tests" = { }; + + "nix-expr" = { }; + "nix-expr-c" = { }; + "nix-expr-test-support" = { }; + "nix-expr-tests" = { }; + + "nix-flake" = { }; + "nix-flake-tests" = { }; + + "nix-main" = { }; + "nix-main-c" = { }; + + "nix-cmd" = { }; + + "nix-cli" = { }; + + "nix-functional-tests" = { supportsCross = false; }; + + "nix-perl-bindings" = { supportsCross = false; }; + "nix-ng" = { }; } - (pkgName: {}: { + (pkgName: { supportsCross ? true }: { # These attributes go right into `packages.`. "${pkgName}" = nixpkgsFor.${system}.native.nixComponents.${pkgName}; "${pkgName}-static" = nixpkgsFor.${system}.static.nixComponents.${pkgName}; } - // flatMapAttrs (lib.genAttrs crossSystems (_: { })) (crossSystem: {}: { + // lib.optionalAttrs supportsCross (flatMapAttrs (lib.genAttrs crossSystems (_: { })) (crossSystem: {}: { # These attributes go right into `packages.`. "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}; - }) + })) // flatMapAttrs (lib.genAttrs stdenvs (_: { })) (stdenvName: {}: { # These attributes go right into `packages.`. "${pkgName}-${stdenvName}" = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nixComponents.${pkgName}; @@ -259,10 +281,10 @@ dockerImage = let pkgs = nixpkgsFor.${system}.native; - image = import ./docker.nix { inherit pkgs; tag = version; }; + image = import ./docker.nix { inherit pkgs; tag = pkgs.nix.version; }; in pkgs.runCommand - "docker-image-tarball-${version}" + "docker-image-tarball-${pkgs.nix.version}" { meta.description = "Docker image with Nix for ${system}"; } '' mkdir -p $out/nix-support @@ -330,6 +352,7 @@ ++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.nativeBuildInputs ++ pkgs.nixComponents.nix-internal-api-docs.nativeBuildInputs ++ pkgs.nixComponents.nix-external-api-docs.nativeBuildInputs + ++ pkgs.nixComponents.nix-functional-tests.baseNativeBuildInputs ++ lib.optional (!stdenv.buildPlatform.canExecute stdenv.hostPlatform # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479 @@ -340,6 +363,7 @@ ++ [ pkgs.buildPackages.cmake pkgs.buildPackages.shellcheck + pkgs.buildPackages.changelog-d modular.pre-commit.settings.package (pkgs.writeScriptBin "pre-commit-hooks-install" modular.pre-commit.settings.installationScript) diff --git a/maintainers/data/release-credits-email-to-handle.json b/maintainers/data/release-credits-email-to-handle.json new file mode 100644 index 000000000..cddc1a6e7 --- /dev/null +++ b/maintainers/data/release-credits-email-to-handle.json @@ -0,0 +1,52 @@ +{ + "bogus": "bogus", + "edolstra@gmail.com": "edolstra", + "roberth@users.noreply.github.com": "roberth", + "toscano.pino@tiscali.it": "pinotree", + "valentin@gagarin.work": "fricklerhandwerk", + "mr.trubach@icloud.com": "tie", + "robert@roberthensing.nl": "roberth", + "lix@jade.fyi": "lf-", + "cole.e.helbling@outlook.com": "cole-h", + "joerg@thalheim.io": "Mic92", + "John.Ericson@Obsidian.Systems": "Ericson2314", + "ryan.hendrickson@alum.mit.edu": "rhendric", + "67135060+poweredbypie@users.noreply.github.com": "poweredbypie", + "detroyejr@outlook.com": "detroyejr", + "silvan.mosberger@tweag.io": "infinisil", + "vcs@emily.moe": "emilazy", + "farid.m.zakaria@gmail.com": "fzakaria", + "22859658+RTUnreal@users.noreply.github.com": "RTUnreal", + "me@las.rs": "L-as", + "philip.taron@gmail.com": "philiptaron", + "root@goldstein.rs": "GoldsteinE", + "tomberek@users.noreply.github.com": "tomberek", + "lexi.mattick@neuralink.com": "kognise", + "andrew@johnandrewmarshall.com": "amarshall", + "contact@romain-neil.fr": "romain-neil", + "Mic92@users.noreply.github.com": "Mic92", + "valentin.gagarin@tweag.io": "fricklerhandwerk", + "siddhantk232@gmail.com": "siddhantk232", + "kn@openbsd.org": "klemensn", + "slyich@gmail.com": "trofi", + "theophane.hufschmitt@tweag.io": "thufschmitt", + "alicebob@lijzij.de": "alicebob", + "winter@winter.cafe": "winterqt", + "brian@brianmckenna.org": "puffnfresh", + "git@haenoe.party": "haenoe", + "peshogo@gmail.com": "pineapplehunter", + "poweredbypie@users.noreply.github.com": "poweredbypie", + "arthur200126@gmail.com": "Artoria2e5", + "tomberek@gmail.com": "tomberek", + "jaredbaur@fastmail.com": "jmbaur", + "andreas@rammhold.de": "andir", + "hamirmahal@gmail.com": "hamirmahal", + "git@JohnEricson.me": "Ericson2314", + "8763518+SkamDart@users.noreply.github.com": "SkamDart", + "kirillrdy@gmail.com": "kirillrdy", + "pennae@lix.systems": "pennae", + "delroth@gmail.com": "delroth", + "enno@nerdworks.de": "elohmeier", + "mjbauer95@gmail.com": "matthewbauer", + "MostAwesomeDude@gmail.com": "MostAwesomeDude" +} \ No newline at end of file diff --git a/maintainers/data/release-credits-handle-to-name.json b/maintainers/data/release-credits-handle-to-name.json new file mode 100644 index 000000000..abf9ed05b --- /dev/null +++ b/maintainers/data/release-credits-handle-to-name.json @@ -0,0 +1,45 @@ +{ + "fzakaria": "Farid Zakaria", + "kognise": "Lexi Mattick", + "L-as": "Las Safin", + "haenoe": "HaeNoe", + "andir": "Andreas Rammhold", + "matthewbauer": "Matthew Bauer", + "emilazy": "Emily", + "pineapplehunter": "Shogo Takata", + "RTUnreal": null, + "jmbaur": "Jared Baur", + "Ericson2314": "John Ericson", + "pinotree": "Pino Toscano", + "tie": "Ivan Trubach", + "poweredbypie": null, + "fricklerhandwerk": "Valentin Gagarin", + "Mic92": "J\u00f6rg Thalheim", + "alicebob": "Harmen", + "elohmeier": "Enno Richter", + "delroth": "Pierre Bourdon", + "kirillrdy": null, + "thufschmitt": "Th\u00e9ophane Hufschmitt", + "detroyejr": "Jonathan De Troye", + "klemensn": "Klemens Nanni", + "tomberek": null, + "rhendric": "Ryan Hendrickson", + "philiptaron": "Philip Taron", + "puffnfresh": "Brian McKenna", + "lf-": "jade", + "romain-neil": "Romain Neil", + "hamirmahal": "Hamir Mahal", + "edolstra": "Eelco Dolstra", + "Artoria2e5": "Mingye Wang", + "SkamDart": "Cameron", + "roberth": "Robert Hensing", + "amarshall": "Andrew Marshall", + "trofi": "Sergei Trofimovich", + "cole-h": "Cole Helbling", + "infinisil": "Silvan Mosberger", + "siddhantk232": "Siddhant Kumar", + "winterqt": "Winter", + "GoldsteinE": "Max \u201cGoldstein\u201d Siling", + "pennae": null, + "MostAwesomeDude": "Corbin Simpson" +} \ No newline at end of file diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index b5c7bfd53..0b83e5696 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -2,7 +2,7 @@ { imports = [ - inputs.pre-commit-hooks.flakeModule + inputs.git-hooks-nix.flakeModule ]; perSystem = { config, pkgs, ... }: { @@ -275,7 +275,6 @@ ''^src/libutil/current-process\.hh$'' ''^src/libutil/english\.cc$'' ''^src/libutil/english\.hh$'' - ''^src/libutil/environment-variables\.cc$'' ''^src/libutil/error\.cc$'' ''^src/libutil/error\.hh$'' ''^src/libutil/exit\.hh$'' @@ -357,7 +356,6 @@ ''^src/libutil/util\.cc$'' ''^src/libutil/util\.hh$'' ''^src/libutil/variant-wrapper\.hh$'' - ''^src/libutil/windows/environment-variables\.cc$'' ''^src/libutil/windows/file-descriptor\.cc$'' ''^src/libutil/windows/file-path\.cc$'' ''^src/libutil/windows/processes\.cc$'' @@ -485,7 +483,6 @@ ''^tests/unit/libutil/pool\.cc'' ''^tests/unit/libutil/references\.cc'' ''^tests/unit/libutil/suggestions\.cc'' - ''^tests/unit/libutil/tests\.cc'' ''^tests/unit/libutil/url\.cc'' ''^tests/unit/libutil/xml-writer\.cc'' ]; @@ -644,19 +641,8 @@ ''^tests/functional/selfref-gc\.sh$'' ''^tests/functional/shell\.sh$'' ''^tests/functional/shell\.shebang\.sh$'' - ''^tests/functional/signing\.sh$'' ''^tests/functional/simple\.builder\.sh$'' - ''^tests/functional/simple\.sh$'' - ''^tests/functional/ssh-relay\.sh$'' - ''^tests/functional/store-info\.sh$'' - ''^tests/functional/structured-attrs\.sh$'' - ''^tests/functional/substitute-with-invalid-ca\.sh$'' - ''^tests/functional/suggestions\.sh$'' ''^tests/functional/supplementary-groups\.sh$'' - ''^tests/functional/tarball\.sh$'' - ''^tests/functional/test-infra\.sh$'' - ''^tests/functional/test-libstoreconsumer\.sh$'' - ''^tests/functional/timeout\.sh$'' ''^tests/functional/toString-path\.sh$'' ''^tests/functional/user-envs-migration\.sh$'' ''^tests/functional/user-envs-test-case\.sh$'' diff --git a/maintainers/release-credits b/maintainers/release-credits new file mode 100755 index 000000000..7a5c87d7d --- /dev/null +++ b/maintainers/release-credits @@ -0,0 +1,184 @@ +#!/usr/bin/env nix +# vim: set filetype=python: +#!nix develop --impure --expr +#!nix `` +#!nix let flake = builtins.getFlake ("git+file://" + toString ../.); +#!nix pkgs = flake.inputs.nixpkgs.legacyPackages.${builtins.currentSystem}; +#!nix in pkgs.mkShell { nativeBuildInputs = [ +#!nix (pkgs.python3.withPackages (ps: with ps; [ requests ])) +#!nix ]; } +#!nix `` --command python3 + +# This script lists out the contributors for a given release. +# It must be run from the root of the Nix repository. + +import os +import sys +import json +import requests + +github_token = os.environ.get("GITHUB_TOKEN") +if not github_token: + print("GITHUB_TOKEN is not set. If you hit the rate limit, set it", file=sys.stderr) + # Might be ok, as we have a cache. + # raise ValueError("GITHUB_TOKEN must be set") + +# 1. Read the current version in .version +version = os.environ.get("VERSION") +if not version: + version = open(".version").read().strip() + +print(f"Generating release credits for Nix {version}", file=sys.stderr) + +# 2. Compute previous version +vcomponents = version.split(".") +if len(vcomponents) >= 2: + prev_version = f"{vcomponents[0]}.{int(vcomponents[1])-1}.0" +else: + raise ValueError(".version must have at least two components") + +# For unreleased versions +endref = "HEAD" +# For older releases +# endref = version + +# 2. Find the merge base between the current version and the previous version +mergeBase = os.popen(f"git merge-base {prev_version} {endref}").read().strip() +print(f"Merge base between {prev_version} and {endref} is {mergeBase}", file=sys.stderr) + +# 3. Find the date of the merge base +mergeBaseDate = os.popen(f"git show -s --format=%ci {mergeBase}").read().strip()[0:10] +print(f"Merge base date is {mergeBaseDate}", file=sys.stderr) + +# 4. Get the commits between the merge base and the current version + +def get_commits(): + raw = os.popen(f"git log --pretty=format:'%H\t%an\t%ae' {mergeBase}..{endref}").read().strip() + lines = raw.split("\n") + return [ { "hash": items[0], "author": items[1], "email": items[2] } + for line in lines + for items in (line.split("\t"),) + ] + +def commits_to_first_commit_by_email(commits): + by_email = dict() + for commit in commits: + email = commit["email"] + if email not in by_email: + by_email[email] = commit + return by_email + + +samples = commits_to_first_commit_by_email(get_commits()) + +# For quick testing, only pick two samples from the dict +# samples = dict(list(samples.items())[:2]) + +# Query the GitHub API to get handle +def get_github_commit(commit): + url = f"https://api.github.com/repos/NixOS/nix/commits/{commit['hash']}" + headers = {'Authorization': f'token {github_token}'} + response = requests.get(url, headers=headers) + response.raise_for_status() + return response.json() + +class Cache: + def __init__(self, filename, require = True): + self.filename = filename + try: + with open(filename, "r") as f: + self.values = json.load(f) + except FileNotFoundError: + if require: + raise + self.values = dict() + def save(self): + with open(self.filename, "w") as f: + json.dump(self.values, f, indent=4) + print(f"Saved cache to {self.filename}", file=sys.stderr) + +# The email to handle cache maps email addresses to either +# - a handle (string) +# - None (if no handle was found) +email_to_handle_cache = Cache("maintainers/data/release-credits-email-to-handle.json") + +handles = set() +emails = dict() + +for sample in samples: + s = samples[sample] + email = s["email"] + if not email in email_to_handle_cache.values: + print(f"Querying GitHub API for {s['hash']}, to get handle for {s['email']}") + ghc = get_github_commit(samples[sample]) + gha = ghc["author"] + if gha and gha["login"]: + handle = gha["login"] + print(f"Handle: {handle}") + email_to_handle_cache.values[email] = handle + else: + print(f"Found no handle for {s['email']}") + email_to_handle_cache.values[email] = None + handle = email_to_handle_cache.values[email] + if handle is not None: + handles.add(handle) + else: + emails[email] = s["author"] + +# print(email_to_handle_cache.values) + +email_to_handle_cache.save() + +handle_to_name_cache = Cache("maintainers/data/release-credits-handle-to-name.json") + +print(f"Found {len(handles)} handles", file=sys.stderr) + +for handle in handles: + if not handle in handle_to_name_cache.values: + print(f"Querying GitHub API for {handle}, to get name", file=sys.stderr) + url = f"https://api.github.com/users/{handle}" + headers = {'Authorization': f'token {github_token}'} + response = requests.get(url, headers=headers) + response.raise_for_status() + user = response.json() + name = user["name"] + print(f"Name: {name}", file=sys.stderr) + handle_to_name_cache.values[handle] = name + +handle_to_name_cache.save() + +entries = list() + +def is_bot(handle): + return handle.endswith("[bot]") + +for handle in handles: + name = handle_to_name_cache.values[handle] + if name is None: + # This way it looks more regular + name = handle + + # Note to bots: I'm only doing this because it would distract humans, and + # therefore delay your creation. I'm sorry; it's for your benefit. + if not is_bot(handle): + entries += [ f"- {name} [**(@{handle})**](https://github.com/{handle})" ] + +def shuffle(entries): + salt = os.urandom(16) + return sorted(entries, key=lambda x: hash((x, salt))) + +# Fair ordering is undecidable +entries = shuffle(entries) + +# For a sanity check, we could sort the entries by handle instead. +# entries = sorted(entries) + +print("") +print(f"This release was made possible by the following {len(entries)} contributors:") +print("") + +for entry in entries: + print(entry) + +for email in emails: + print(f"- {emails[email]}") diff --git a/maintainers/release-notes b/maintainers/release-notes index 0fca5abf2..c0c4ee734 100755 --- a/maintainers/release-notes +++ b/maintainers/release-notes @@ -1,4 +1,5 @@ #!/usr/bin/env nix +# vim: set filetype=bash: #!nix shell .#changelog-d --command bash # --- CONFIGURATION --- @@ -151,6 +152,13 @@ section_title="Release $version_full ($DATE)" echo "# $section_title" echo changelog-d doc/manual/rl-next | sed -e 's/ *$//' + + if ! $IS_PATCH; then + echo + echo "# Contributors" + echo + VERSION=$version_full ./maintainers/release-credits + fi ) | tee -a $file log "Wrote $file" diff --git a/maintainers/release-process.md b/maintainers/release-process.md index da6886ea9..7a2b3c0a7 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -39,6 +39,10 @@ release: * Proof-read / edit / rearrange the release notes if needed. Breaking changes and highlights should go to the top. +* Run `maintainers/release-credits` to make sure the credits script works + and produces a sensible output. Some emails might not automatically map to + a GitHub handle. + * Push. ```console diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 4c4e2bd6f..731988568 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -42,7 +42,7 @@ my $flakeUrl = $evalInfo->{flake}; my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die) if $flakeUrl; my $nixRev = ($flakeInfo ? $flakeInfo->{revision} : $evalInfo->{jobsetevalinputs}->{nix}->{revision}) or die; -my $buildInfo = decode_json(fetch("$evalUrl/job/build.x86_64-linux", 'application/json')); +my $buildInfo = decode_json(fetch("$evalUrl/job/build.nix.x86_64-linux", 'application/json')); #print Dumper($buildInfo); my $releaseName = $buildInfo->{nixname}; @@ -91,7 +91,7 @@ sub getStorePath { sub copyManual { my $manual; eval { - $manual = getStorePath("build.x86_64-linux", "doc"); + $manual = getStorePath("build.nix.x86_64-linux", "doc"); }; if ($@) { warn "$@"; @@ -240,12 +240,12 @@ if ($haveDocker) { # Upload nix-fallback-paths.nix. write_file("$tmpDir/fallback-paths.nix", "{\n" . - " x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" . - " i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" . - " aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" . - " riscv64-linux = \"" . getStorePath("buildCross.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" . - " x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" . - " aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" . + " x86_64-linux = \"" . getStorePath("build.nix.x86_64-linux") . "\";\n" . + " i686-linux = \"" . getStorePath("build.nix.i686-linux") . "\";\n" . + " aarch64-linux = \"" . getStorePath("build.nix.aarch64-linux") . "\";\n" . + " riscv64-linux = \"" . getStorePath("buildCross.nix.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" . + " x86_64-darwin = \"" . getStorePath("build.nix.x86_64-darwin") . "\";\n" . + " aarch64-darwin = \"" . getStorePath("build.nix.aarch64-darwin") . "\";\n" . "}\n"); # Upload release files to S3. diff --git a/meson.build b/meson.build index 1554244ab..715a3862d 100644 --- a/meson.build +++ b/meson.build @@ -42,3 +42,4 @@ subproject('nix-fetchers-tests') subproject('nix-expr-test-support') subproject('nix-expr-tests') subproject('nix-flake-tests') +subproject('nix-functional-tests') diff --git a/mk/platform.mk b/mk/platform.mk index fe960dedf..22c114a20 100644 --- a/mk/platform.mk +++ b/mk/platform.mk @@ -29,4 +29,8 @@ ifdef HOST_OS HOST_SOLARIS = 1 HOST_UNIX = 1 endif + ifeq ($(HOST_KERNEL), gnu) + HOST_HURD = 1 + HOST_UNIX = 1 + endif endif diff --git a/mk/run-test.sh b/mk/run-test.sh index 543c845e1..7f9f1d5f8 100755 --- a/mk/run-test.sh +++ b/mk/run-test.sh @@ -28,7 +28,7 @@ run_test if [[ "$status" = 0 ]]; then echo "$post_run_msg [${green}PASS$normal]" -elif [[ "$status" = 99 ]]; then +elif [[ "$status" = 77 ]]; then echo "$post_run_msg [${yellow}SKIP$normal]" else echo "$post_run_msg [${red}FAIL$normal]" diff --git a/package.nix b/package.nix index 86ca6c9e9..e3c75530b 100644 --- a/package.nix +++ b/package.nix @@ -47,15 +47,12 @@ , pname ? "nix-super" -, versionSuffix ? "" +, version +, versionSuffix # Whether to build Nix. Useful to skip for tasks like testing existing pre-built versions of Nix , doBuild ? true -# Run the unit tests as part of the build. See `installUnitTests` for an -# alternative to this. -, doCheck ? __forDefaults.canRunInstalled - # Run the functional tests as part of the build. , doInstallCheck ? test-client != null || __forDefaults.canRunInstalled @@ -88,11 +85,6 @@ # - readline , readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline" -# Whether to install unit tests. This is useful when cross compiling -# since we cannot run them natively during the build, but can do so -# later. -, installUnitTests ? doBuild && !__forDefaults.canExecuteHost - # For running the functional tests against a pre-built Nix. Probably # want to use in conjunction with `doBuild = false;`. , test-daemon ? null @@ -112,13 +104,11 @@ let inherit (lib) fileset; - version = lib.fileContents ./.version + versionSuffix; - # selected attributes with defaults, will be used to define some # things which should instead be gotten via `finalAttrs` in order to # work with overriding. attrs = { - inherit doBuild doCheck doInstallCheck; + inherit doBuild doInstallCheck; }; mkDerivation = @@ -134,16 +124,11 @@ in mkDerivation (finalAttrs: let inherit (finalAttrs) - doCheck doInstallCheck ; doBuild = !finalAttrs.dontBuild; - # Either running the unit tests during the build, or installing them - # to be run later, requiresthe unit tests to be built. - buildUnitTests = doCheck || installUnitTests; - in { inherit pname version; @@ -177,8 +162,6 @@ in { ./scripts/local.mk ] ++ lib.optionals enableManual [ ./doc/manual - ] ++ lib.optionals buildUnitTests [ - ./tests/unit ] ++ lib.optionals doInstallCheck [ ./tests/functional ])); @@ -191,8 +174,6 @@ in { # If we are doing just build or just docs, the one thing will use # "out". We only need additional outputs if we are doing both. ++ lib.optional (doBuild && enableManual) "doc" - ++ lib.optional installUnitTests "check" - ++ lib.optional doCheck "testresults" ; nativeBuildInputs = [ @@ -216,7 +197,8 @@ in { ] ++ lib.optional stdenv.hostPlatform.isStatic unixtools.hexdump ; - buildInputs = lib.optionals doBuild [ + buildInputs = lib.optionals doBuild ( + [ brotli bzip2 curl @@ -230,41 +212,30 @@ in { ({ inherit readline editline; }.${readlineFlavor}) ] ++ lib.optionals enableMarkdown [ lowdown - ] ++ lib.optionals buildUnitTests [ - gtest - rapidcheck ] ++ lib.optional stdenv.isLinux libseccomp ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid # There have been issues building these dependencies ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin)) - (aws-sdk-cpp.override { - apis = ["s3" "transfer"]; - customMemoryManagement = false; - }) - ; + aws-sdk-cpp + ); - propagatedBuildInputs = [ + propagatedBuildInputs = lib.optionals doBuild ([ boost nlohmann_json - ] ++ lib.optional enableGC boehmgc; + ] ++ lib.optional enableGC boehmgc + ); dontBuild = !attrs.doBuild; - doCheck = attrs.doCheck; configureFlags = [ (lib.enableFeature doBuild "build") - (lib.enableFeature buildUnitTests "unit-tests") (lib.enableFeature doInstallCheck "functional-tests") (lib.enableFeature enableManual "doc-gen") (lib.enableFeature enableGC "gc") (lib.enableFeature enableMarkdown "markdown") - (lib.enableFeature installUnitTests "install-unit-tests") (lib.withFeatureAs true "readline-flavor" readlineFlavor) ] ++ lib.optionals (!forDevShell) [ "--sysconfdir=/etc" - ] ++ lib.optionals installUnitTests [ - "--with-check-bin-dir=${builtins.placeholder "check"}/bin" - "--with-check-lib-dir=${builtins.placeholder "check"}/lib" ] ++ lib.optionals (doBuild) [ "--with-boost=${boost}/lib" ] ++ lib.optionals (doBuild && stdenv.isLinux) [ @@ -345,10 +316,6 @@ in { platforms = lib.platforms.unix ++ lib.platforms.windows; mainProgram = "nix"; broken = !(lib.all (a: a) [ - # We cannot run or install unit tests if we don't build them or - # Nix proper (which they depend on). - (installUnitTests -> doBuild) - (doCheck -> doBuild) # The build process for the manual currently requires extracting # data from the Nix executable we are trying to document. (enableManual -> doBuild) diff --git a/packaging/components.nix b/packaging/components.nix index 870e9ae61..5fc3236cf 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -1,11 +1,34 @@ +{ + lib, + src, + officialRelease, +}: + scope: + let inherit (scope) callPackage; + + baseVersion = lib.fileContents ../.version; + + versionSuffix = lib.optionalString (!officialRelease) "pre"; + + fineVersionSuffix = lib.optionalString + (!officialRelease) + "pre${builtins.substring 0 8 (src.lastModifiedDate or src.lastModified or "19700101")}_${src.shortRev or "dirty"}"; + + fineVersion = baseVersion + fineVersionSuffix; in # This becomes the pkgs.nixComponents attribute set { - nix = callPackage ../package.nix { }; + version = baseVersion + versionSuffix; + inherit versionSuffix; + + nix = callPackage ../package.nix { + version = fineVersion; + versionSuffix = fineVersionSuffix; + }; nix-util = callPackage ../src/libutil/package.nix { }; nix-util-c = callPackage ../src/libutil-c/package.nix { }; @@ -33,11 +56,15 @@ in nix-cmd = callPackage ../src/libcmd/package.nix { }; - # Will replace `nix` once the old build system is gone. - nix-ng = callPackage ../src/nix/package.nix { }; + nix-cli = callPackage ../src/nix/package.nix { version = fineVersion; }; - nix-internal-api-docs = callPackage ../src/internal-api-docs/package.nix { }; - nix-external-api-docs = callPackage ../src/external-api-docs/package.nix { }; + nix-functional-tests = callPackage ../src/nix-functional-tests/package.nix { version = fineVersion; }; + + nix-internal-api-docs = callPackage ../src/internal-api-docs/package.nix { version = fineVersion; }; + nix-external-api-docs = callPackage ../src/external-api-docs/package.nix { version = fineVersion; }; nix-perl-bindings = callPackage ../src/perl/package.nix { }; + + # Will replace `nix` once the old build system is gone. + nix-ng = callPackage ../packaging/everything.nix { }; } diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index f09ca5d18..21c48e5cc 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -8,7 +8,6 @@ pkgs, stdenv, - versionSuffix, }: let @@ -63,14 +62,28 @@ let # Work around weird `--as-needed` linker behavior with BSD, see # https://github.com/mesonbuild/meson/issues/3593 - bsdNoLinkAsNeeded = finalAttrs: prevAttrs: lib.optionalAttrs stdenv.hostPlatform.isBSD { - mesonFlags = [ (lib.mesonBool "b_asneeded" false) ] ++ prevAttrs.mesonFlags or []; - }; + bsdNoLinkAsNeeded = finalAttrs: prevAttrs: + lib.optionalAttrs stdenv.hostPlatform.isBSD { + mesonFlags = [ (lib.mesonBool "b_asneeded" false) ] ++ prevAttrs.mesonFlags or []; + }; + miscGoodPractice = finalAttrs: prevAttrs: + { + strictDeps = prevAttrs.strictDeps or true; + enableParallelBuilding = true; + }; in scope: { - inherit stdenv versionSuffix; - version = lib.fileContents ../.version + versionSuffix; + inherit stdenv; + + aws-sdk-cpp = (pkgs.aws-sdk-cpp.override { + apis = [ "s3" "transfer" ]; + customMemoryManagement = false; + }).overrideAttrs { + # only a stripped down version is built, which takes a lot less resources + # to build, so we don't need a "big-parallel" machine. + requiredSystemFeatures = [ ]; + }; libseccomp = pkgs.libseccomp.overrideAttrs (_: rec { version = "2.5.5"; @@ -136,8 +149,14 @@ scope: { inherit resolvePath filesetToSource; - mkMesonDerivation = f: stdenv.mkDerivation - (lib.extends - (lib.composeExtensions bsdNoLinkAsNeeded localSourceLayer) - f); + mkMesonDerivation = f: let + exts = [ + miscGoodPractice + bsdNoLinkAsNeeded + localSourceLayer + ]; + in stdenv.mkDerivation + (lib.extends + (lib.foldr lib.composeExtensions (_: _: {}) exts) + f); } diff --git a/packaging/everything.nix b/packaging/everything.nix new file mode 100644 index 000000000..8c8ce6611 --- /dev/null +++ b/packaging/everything.nix @@ -0,0 +1,93 @@ +{ + lib, + stdenv, + buildEnv, + + nix-util, + nix-util-c, + nix-util-test-support, + nix-util-tests, + + nix-store, + nix-store-c, + nix-store-test-support, + nix-store-tests, + + nix-fetchers, + nix-fetchers-tests, + + nix-expr, + nix-expr-c, + nix-expr-test-support, + nix-expr-tests, + + nix-flake, + nix-flake-tests, + + nix-main, + nix-main-c, + + nix-cmd, + + nix-cli, + + nix-functional-tests, + + nix-internal-api-docs, + nix-external-api-docs, + + nix-perl-bindings, +}: + +(buildEnv rec { + name = "nix-${nix-cli.version}"; + paths = [ + nix-util + nix-util-c + nix-util-test-support + nix-util-tests + + nix-store + nix-store-c + nix-store-test-support + nix-store-tests + + nix-fetchers + nix-fetchers-tests + + nix-expr + nix-expr-c + nix-expr-test-support + nix-expr-tests + + nix-flake + nix-flake-tests + + nix-main + nix-main-c + + nix-cmd + + nix-cli + + nix-internal-api-docs + nix-external-api-docs + + ] ++ lib.optionals (stdenv.buildPlatform.canExecute stdenv.hostPlatform) [ + nix-perl-bindings + ]; +}).overrideAttrs (_: { + doCheck = true; + doInstallCheck = true; + + checkInputs = [ + # Actually run the unit tests too + nix-util-tests.tests.run + nix-store-tests.tests.run + nix-expr-tests.tests.run + nix-flake-tests.tests.run + ]; + installCheckInputs = [ + nix-functional-tests + ]; +}) diff --git a/packaging/hydra.nix b/packaging/hydra.nix index dbe992476..65978835c 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -6,6 +6,7 @@ , linux64BitSystems , nixpkgsFor , self +, officialRelease }: let inherit (inputs) nixpkgs nixpkgs-regression; @@ -16,7 +17,7 @@ let }; testNixVersions = pkgs: client: daemon: - pkgs.callPackage ../package.nix { + pkgs.nixComponents.callPackage ../package.nix { pname = "nix-tests" + lib.optionalString @@ -28,6 +29,12 @@ let test-daemon = daemon; doBuild = false; + + # This could be more accurate, but a shorter version will match the + # fine version with rev. This functionality is already covered in + # the normal test, so it's fine. + version = pkgs.nixComponents.version; + versionSuffix = pkgs.nixComponents.versionSuffix; }; # Technically we could just return `pkgs.nixComponents`, but for Hydra it's @@ -54,6 +61,8 @@ let "nix-main" "nix-main-c" "nix-cmd" + "nix-cli" + "nix-functional-tests" "nix-ng" ]; in @@ -68,14 +77,16 @@ in lib.genAttrs linux64BitSystems (system: nixpkgsFor.${system}.static.nixComponents.${pkgName})); buildCross = forAllPackages (pkgName: - forAllCrossSystems (crossSystem: - lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}))); + # Hack to avoid non-evaling package + (if pkgName == "nix-functional-tests" then lib.flip builtins.removeAttrs ["x86_64-w64-mingw32"] else lib.id) + (forAllCrossSystems (crossSystem: + lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName})))); buildNoGc = forAllSystems (system: self.packages.${system}.nix.override { enableGC = false; } ); - buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.nix_noTests); + buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.nix-cli); # Toggles some settings for better coverage. Windows needs these # library combinations, and Debian build Nix with GNU readline too. diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index a0a404e57..82ad7d862 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -101,7 +101,7 @@ static int main_build_remote(int argc, char * * argv) } std::optional drvPath; - StoreReference storeUri; + std::string storeUri; while (true) { @@ -234,17 +234,16 @@ static int main_build_remote(int argc, char * * argv) lock = -1; try { + storeUri = bestMachine->storeUri.render(); - Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri.render())); + Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", storeUri)); sshStore = bestMachine->openStore(); sshStore->connect(); - storeUri = bestMachine->storeUri; - } catch (std::exception & e) { auto msg = chomp(drainFD(5, false)); printError("cannot build on '%s': %s%s", - bestMachine->storeUri.render(), e.what(), + storeUri, e.what(), msg.empty() ? "" : ": " + msg); bestMachine->enabled = false; continue; @@ -259,15 +258,28 @@ connected: assert(sshStore); - std::cerr << "# accept\n" << storeUri.render() << "\n"; + std::cerr << "# accept\n" << storeUri << "\n"; auto inputs = readStrings(source); auto wantedOutputs = readStrings(source); - AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri.render()) + ".upload-lock", true); + AutoCloseFD uploadLock; + { + auto setUpdateLock = [&](auto && fileName){ + uploadLock = openLockFile(currentLoad + "/" + escapeUri(fileName) + ".upload-lock", true); + }; + try { + setUpdateLock(storeUri); + } catch (SysError & e) { + if (e.errNo != ENAMETOOLONG) throw; + // Try again hashing the store URL so we have a shorter path + auto h = hashString(HashAlgorithm::MD5, storeUri); + setUpdateLock(h.to_string(HashFormat::Base64, false)); + } + } { - Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri.render())); + Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri)); auto old = signal(SIGALRM, handleAlarm); alarm(15 * 60); @@ -280,7 +292,7 @@ connected: auto substitute = settings.buildersUseSubstitutes ? Substitute : NoSubstitute; { - Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri.render())); + Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri)); copyPaths(*store, *sshStore, store->parseStorePathSet(inputs), NoRepair, NoCheckSigs, substitute); } @@ -318,7 +330,7 @@ connected: optResult = sshStore->buildDerivation(*drvPath, (const BasicDerivation &) drv); auto & result = *optResult; if (!result.success()) - throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri.render(), result.errorMsg); + throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg); } else { copyClosure(*store, *sshStore, StorePathSet {*drvPath}, NoRepair, NoCheckSigs, substitute); auto res = sshStore->buildPathsWithResults({ @@ -361,7 +373,7 @@ connected: } if (!missingPaths.empty()) { - Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri.render())); + Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri)); if (auto localStore = store.dynamic_pointer_cast()) for (auto & path : missingPaths) localStore->locksHeld.insert(store->printStorePath(path)); /* FIXME: ugly */ diff --git a/src/external-api-docs/package.nix b/src/external-api-docs/package.nix index da136bbe1..743b3e9b7 100644 --- a/src/external-api-docs/package.nix +++ b/src/external-api-docs/package.nix @@ -53,10 +53,6 @@ mkMesonDerivation (finalAttrs: { echo "doc external-api-docs $out/share/doc/nix/external-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products ''; - enableParallelBuilding = true; - - strictDeps = true; - meta = { platforms = lib.platforms.all; }; diff --git a/src/internal-api-docs/package.nix b/src/internal-api-docs/package.nix index f2077dcaf..07ca6d4d9 100644 --- a/src/internal-api-docs/package.nix +++ b/src/internal-api-docs/package.nix @@ -48,10 +48,6 @@ mkMesonDerivation (finalAttrs: { echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products ''; - enableParallelBuilding = true; - - strictDeps = true; - meta = { platforms = lib.platforms.all; }; diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index decadd751..fcef92487 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -91,75 +91,11 @@ MixEvalArgs::MixEvalArgs() .longName = "include", .shortName = 'I', .description = R"( - Add *path* to the Nix search path. The Nix search path is - initialized from the colon-separated [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) environment - variable, and is used to look up the location of Nix expressions using [paths](@docroot@/language/types.md#type-path) enclosed in angle - brackets (i.e., ``). + Add *path* to search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md) - For instance, passing + This option may be given multiple times. - ``` - -I /home/eelco/Dev - -I /etc/nixos - ``` - - will cause Nix to look for paths relative to `/home/eelco/Dev` and - `/etc/nixos`, in that order. This is equivalent to setting the - `NIX_PATH` environment variable to - - ``` - /home/eelco/Dev:/etc/nixos - ``` - - It is also possible to match paths against a prefix. For example, - passing - - ``` - -I nixpkgs=/home/eelco/Dev/nixpkgs-branch - -I /etc/nixos - ``` - - will cause Nix to search for `` in - `/home/eelco/Dev/nixpkgs-branch/path` and `/etc/nixos/nixpkgs/path`. - - If a path in the Nix search path starts with `http://` or `https://`, - it is interpreted as the URL of a tarball that will be downloaded and - unpacked to a temporary location. The tarball must consist of a single - top-level directory. For example, passing - - ``` - -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/master.tar.gz - ``` - - tells Nix to download and use the current contents of the `master` - branch in the `nixpkgs` repository. - - The URLs of the tarballs from the official `nixos.org` channels - (see [the manual page for `nix-channel`](../nix-channel.md)) can be - abbreviated as `channel:`. For instance, the - following two flags are equivalent: - - ``` - -I nixpkgs=channel:nixos-21.05 - -I nixpkgs=https://nixos.org/channels/nixos-21.05/nixexprs.tar.xz - ``` - - You can also fetch source trees using [flake URLs](./nix3-flake.md#url-like-syntax) and add them to the - search path. For instance, - - ``` - -I nixpkgs=flake:nixpkgs - ``` - - specifies that the prefix `nixpkgs` shall refer to the source tree - downloaded from the `nixpkgs` entry in the flake registry. Similarly, - - ``` - -I nixpkgs=flake:github:NixOS/nixpkgs/nixos-22.05 - ``` - - makes `` refer to a particular branch of the - `NixOS/nixpkgs` repository on GitHub. + Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH). )", .category = category, .labels = {"path"}, diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index 8796ad5ba..6c9ee6748 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -104,12 +104,12 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() auto drvPath = attr->forceDerivation(); - std::optional priority; + std::optional priority; if (attr->maybeGetAttr(state->sOutputSpecified)) { } else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) { if (auto aPriority = aMeta->maybeGetAttr("priority")) - priority = aPriority->getInt(); + priority = aPriority->getInt().value; } return {{ diff --git a/src/libcmd/installable-value.hh b/src/libcmd/installable-value.hh index 798cb5e1a..60207cd23 100644 --- a/src/libcmd/installable-value.hh +++ b/src/libcmd/installable-value.hh @@ -40,7 +40,7 @@ struct ExtraPathInfoValue : ExtraPathInfo /** * An optional priority for use with "build envs". See Package */ - std::optional priority; + std::optional priority; /** * The attribute path associated with this value. The idea is diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index 88c3f640b..6a0d05d9f 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -1,21 +1,23 @@ #include "markdown.hh" -#include "util.hh" +#include "environment-variables.hh" +#include "error.hh" #include "finally.hh" #include "terminal.hh" #if HAVE_LOWDOWN -# include -# include +# include +# include #endif namespace nix { -std::string renderMarkdownToTerminal(std::string_view markdown) -{ #if HAVE_LOWDOWN +static std::string doRenderMarkdownToTerminal(std::string_view markdown) +{ int windowWidth = getWindowSize().second; - struct lowdown_opts opts { + struct lowdown_opts opts + { .type = LOWDOWN_TERM, .maxdepth = 20, .cols = (size_t) std::max(windowWidth - 5, 60), @@ -51,9 +53,21 @@ std::string renderMarkdownToTerminal(std::string_view markdown) throw Error("allocation error while rendering Markdown"); return filterANSIEscapes(std::string(buf->data, buf->size), !isTTY()); -#else - return std::string(markdown); -#endif } +std::string renderMarkdownToTerminal(std::string_view markdown) +{ + if (auto e = getEnv("_NIX_TEST_RAW_MARKDOWN"); e && *e == "1") + return std::string(markdown); + else + return doRenderMarkdownToTerminal(markdown); } + +#else +std::string renderMarkdownToTerminal(std::string_view markdown) +{ + return std::string(markdown); +} +#endif + +} // namespace nix diff --git a/src/libcmd/markdown.hh b/src/libcmd/markdown.hh index a04d32a4f..66db1736c 100644 --- a/src/libcmd/markdown.hh +++ b/src/libcmd/markdown.hh @@ -1,10 +1,17 @@ #pragma once ///@file -#include "types.hh" +#include namespace nix { +/** + * Render the given Markdown text to the terminal. + * + * If Nix is compiled without Markdown support, this function will return the input text as-is. + * + * The renderer takes into account the terminal width, and wraps text accordingly. + */ std::string renderMarkdownToTerminal(std::string_view markdown); } diff --git a/src/libcmd/package.nix b/src/libcmd/package.nix index ec3aa4660..cde494901 100644 --- a/src/libcmd/package.nix +++ b/src/libcmd/package.nix @@ -93,14 +93,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated - # to work with `strictDeps`. - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc index b285c8a9a..187af46ea 100644 --- a/src/libcmd/repl-interacter.cc +++ b/src/libcmd/repl-interacter.cc @@ -19,6 +19,7 @@ extern "C" { #include "repl-interacter.hh" #include "file-system.hh" #include "repl.hh" +#include "environment-variables.hh" namespace nix { @@ -34,6 +35,7 @@ void sigintHandler(int signo) static detail::ReplCompleterMixin * curRepl; // ugly +#ifndef USE_READLINE static char * completionCallback(char * s, int * match) { auto possible = curRepl->completePrefix(s); @@ -100,6 +102,7 @@ static int listPossibleCallback(char * s, char *** avp) return ac; } +#endif ReadlineLikeInteracter::Guard ReadlineLikeInteracter::init(detail::ReplCompleterMixin * repl) { @@ -175,10 +178,23 @@ bool ReadlineLikeInteracter::getLine(std::string & input, ReplPromptType promptT return true; } + // editline doesn't echo the input to the output when non-interactive, unlike readline + // this results in a different behavior when running tests. The echoing is + // quite useful for reading the test output, so we add it here. + if (auto e = getEnv("_NIX_TEST_REPL_ECHO"); s && e && *e == "1") + { +#ifndef USE_READLINE + // This is probably not right for multi-line input, but we don't use that + // in the characterisation tests, so it's fine. + std::cout << promptForType(promptType) << s << std::endl; +#endif + } + if (!s) return false; input += s; input += '\n'; + return true; } diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 3b056eb10..dbce4ac50 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -7,7 +7,6 @@ #include "ansicolor.hh" #include "shared.hh" -#include "config-global.hh" #include "eval.hh" #include "eval-settings.hh" #include "attr-path.hh" @@ -77,10 +76,14 @@ struct NixRepl int displ; StringSet varNames; + RunNix * runNixPtr; + + void runNix(Path program, const Strings & args, const std::optional & input = {}); + std::unique_ptr interacter; NixRepl(const LookupPath & lookupPath, nix::ref store,ref state, - std::function getValues); + std::function getValues, RunNix * runNix); virtual ~NixRepl() = default; ReplExitStatus mainLoop() override; @@ -125,32 +128,16 @@ std::string removeWhitespace(std::string s) NixRepl::NixRepl(const LookupPath & lookupPath, nix::ref store, ref state, - std::function getValues) + std::function getValues, RunNix * runNix = nullptr) : AbstractNixRepl(state) , debugTraceIndex(0) , getValues(getValues) , staticEnv(new StaticEnv(nullptr, state->staticBaseEnv.get())) + , runNixPtr{runNix} , interacter(make_unique(getDataDir() + "/nix/repl-history")) { } -void runNix(Path program, const Strings & args, - const std::optional & input = {}) -{ - auto subprocessEnv = getEnv(); - subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue(); - //isInteractive avoid grabling interactive commands - runProgram2(RunOptions { - .program = settings.nixBinDir+ "/" + program, - .args = args, - .environment = subprocessEnv, - .input = input, - .isInteractive = true, - }); - - return; -} - static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positions, const DebugTrace & dt) { if (dt.isError) @@ -217,7 +204,7 @@ ReplExitStatus NixRepl::mainLoop() case ProcessLineResult::PromptAgain: break; default: - abort(); + unreachable(); } } catch (ParseError & e) { if (e.msg().find("unexpected end of file") != std::string::npos) { @@ -644,9 +631,6 @@ ProcessLineResult NixRepl::processLine(std::string line) fallbackPos = attr->pos; fallbackDoc = state->getDocCommentForPos(fallbackPos); } - - } else { - evalString(arg, v); } evalString(arg, v); @@ -836,9 +820,18 @@ void NixRepl::evalString(std::string s, Value & v) } +void NixRepl::runNix(Path program, const Strings & args, const std::optional & input) +{ + if (runNixPtr) + (*runNixPtr)(program, args, input); + else + throw Error("Cannot run '%s', no method of calling the Nix CLI provided", program); +} + + std::unique_ptr AbstractNixRepl::create( const LookupPath & lookupPath, nix::ref store, ref state, - std::function getValues) + std::function getValues, RunNix * runNix) { return std::make_unique( lookupPath, diff --git a/src/libcmd/repl.hh b/src/libcmd/repl.hh index 3fd4b2c39..11d1820f5 100644 --- a/src/libcmd/repl.hh +++ b/src/libcmd/repl.hh @@ -19,9 +19,19 @@ struct AbstractNixRepl typedef std::vector> AnnotatedValues; + using RunNix = void(Path program, const Strings & args, const std::optional & input); + + /** + * @param runNix Function to run the nix CLI to support various + * `:` commands. Optional; if not provided, + * everything else will still work fine, but those commands won't. + */ static std::unique_ptr create( - const LookupPath & lookupPath, nix::ref store, ref state, - std::function getValues); + const LookupPath & lookupPath, + nix::ref store, + ref state, + std::function getValues, + RunNix * runNix = nullptr); static ReplExitStatus runSimple( ref evalState, diff --git a/src/libexpr-c/nix_api_expr.h b/src/libexpr-c/nix_api_expr.h index adf8b65b1..1764b49f3 100644 --- a/src/libexpr-c/nix_api_expr.h +++ b/src/libexpr-c/nix_api_expr.h @@ -14,6 +14,16 @@ #include "nix_api_util.h" #include +#ifndef __has_c_attribute +# define __has_c_attribute(x) 0 +#endif + +#if __has_c_attribute(deprecated) +# define NIX_DEPRECATED(msg) [[deprecated(msg)]] +#else +# define NIX_DEPRECATED(msg) +#endif + #ifdef __cplusplus extern "C" { #endif @@ -45,7 +55,7 @@ typedef struct EvalState EvalState; // nix::EvalState * @see nix_value_incref, nix_value_decref */ typedef struct nix_value nix_value; -[[deprecated("use nix_value instead")]] typedef nix_value Value; +NIX_DEPRECATED("use nix_value instead") typedef nix_value Value; // Function prototypes /** diff --git a/src/libexpr-c/nix_api_value.cc b/src/libexpr-c/nix_api_value.cc index 845e87935..fa2a9cbe2 100644 --- a/src/libexpr-c/nix_api_value.cc +++ b/src/libexpr-c/nix_api_value.cc @@ -306,7 +306,7 @@ int64_t nix_get_int(nix_c_context * context, const nix_value * value) try { auto & v = check_value_in(value); assert(v.type() == nix::nInt); - return v.integer(); + return v.integer().value; } NIXC_CATCH_ERRS_RES(0); } diff --git a/src/libexpr-c/package.nix b/src/libexpr-c/package.nix index 0b895437b..eb42195a4 100644 --- a/src/libexpr-c/package.nix +++ b/src/libexpr-c/package.nix @@ -63,12 +63,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index d61d93630..2f67260c5 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -134,7 +134,7 @@ std::pair findPackageFilename(EvalState & state, Value & v return {SourcePath{path.accessor, CanonPath(fn.substr(0, colon))}, lineno}; } catch (std::invalid_argument & e) { fail(); - abort(); + unreachable(); } } diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 26352187e..9019edc1f 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -4,6 +4,8 @@ #include "eval.hh" #include "eval-inline.hh" #include "store-api.hh" +// Need specialization involving `SymbolStr` just in this one module. +#include "strings-inline.hh" namespace nix::eval_cache { @@ -326,7 +328,7 @@ struct AttrDb case AttrType::Bool: return {{rowId, queryAttribute.getInt(2) != 0}}; case AttrType::Int: - return {{rowId, int_t{queryAttribute.getInt(2)}}}; + return {{rowId, int_t{NixInt{queryAttribute.getInt(2)}}}}; case AttrType::ListOfStrings: return {{rowId, tokenizeString>(queryAttribute.getStr(2), "\t")}}; case AttrType::Missing: @@ -469,7 +471,7 @@ Value & AttrCursor::forceValue() else if (v.type() == nBool) cachedValue = {root->db->setBool(getKey(), v.boolean()), v.boolean()}; else if (v.type() == nInt) - cachedValue = {root->db->setInt(getKey(), v.integer()), int_t{v.integer()}}; + cachedValue = {root->db->setInt(getKey(), v.integer().value), int_t{v.integer()}}; else if (v.type() == nAttrs) ; // FIXME: do something? else diff --git a/src/libexpr/eval-gc.cc b/src/libexpr/eval-gc.cc index 97b24e3fb..2f0e8c0c9 100644 --- a/src/libexpr/eval-gc.cc +++ b/src/libexpr/eval-gc.cc @@ -1,5 +1,7 @@ #include "error.hh" #include "environment-variables.hh" +#include "eval-settings.hh" +#include "config-global.hh" #include "serialise.hh" #include "eval-gc.hh" @@ -84,14 +86,17 @@ void fixupBoehmStackPointer(void ** sp_ptr, void * _pthread_id) { void *& sp = *sp_ptr; auto pthread_id = reinterpret_cast(_pthread_id); +# ifndef __APPLE__ pthread_attr_t pattr; +# endif size_t osStackSize; - void * osStackLow; + // The low address of the stack, which grows down. + void * osStackLimit; void * osStackBase; # ifdef __APPLE__ osStackSize = pthread_get_stacksize_np(pthread_id); - osStackLow = pthread_get_stackaddr_np(pthread_id); + osStackLimit = pthread_get_stackaddr_np(pthread_id); # else if (pthread_attr_init(&pattr)) { throw Error("fixupBoehmStackPointer: pthread_attr_init failed"); @@ -110,18 +115,18 @@ void fixupBoehmStackPointer(void ** sp_ptr, void * _pthread_id) # else # error "Need one of `pthread_attr_get_np` or `pthread_getattr_np`" # endif - if (pthread_attr_getstack(&pattr, &osStackLow, &osStackSize)) { + if (pthread_attr_getstack(&pattr, &osStackLimit, &osStackSize)) { throw Error("fixupBoehmStackPointer: pthread_attr_getstack failed"); } if (pthread_attr_destroy(&pattr)) { throw Error("fixupBoehmStackPointer: pthread_attr_destroy failed"); } # endif - osStackBase = (char *) osStackLow + osStackSize; + osStackBase = (char *) osStackLimit + osStackSize; // NOTE: We assume the stack grows down, as it does on all architectures we support. // Architectures that grow the stack up are rare. - if (sp >= osStackBase || sp < osStackLow) { // lo is outside the os stack - sp = osStackLow; + if (sp >= osStackBase || sp < osStackLimit) { // sp is outside the os stack + sp = osStackLimit; } } @@ -206,10 +211,17 @@ static inline void initGCReal() } } +static size_t gcCyclesAfterInit = 0; + +size_t getGCCycles() +{ + assertGCInitialized(); + return static_cast(GC_get_gc_no()) - gcCyclesAfterInit; +} + #endif static bool gcInitialised = false; -static GC_word gcCyclesAfterInit = 0; void initGC() { @@ -218,10 +230,17 @@ void initGC() #if HAVE_BOEHMGC initGCReal(); + + gcCyclesAfterInit = GC_get_gc_no(); #endif + // NIX_PATH must override the regular setting + // See the comment in applyConfig + if (auto nixPathEnv = getEnv("NIX_PATH")) { + globalConfig.set("nix-path", concatStringsSep(" ", EvalSettings::parseNixPath(nixPathEnv.value()))); + } + gcInitialised = true; - gcCyclesAfterInit = GC_get_gc_no(); } void assertGCInitialized() @@ -229,10 +248,4 @@ void assertGCInitialized() assert(gcInitialised); } -size_t getGCCycles() -{ - assertGCInitialized(); - return GC_get_gc_no() - gcCyclesAfterInit; -} - } // namespace nix diff --git a/src/libexpr/eval-gc.hh b/src/libexpr/eval-gc.hh index 76f019729..005175eb7 100644 --- a/src/libexpr/eval-gc.hh +++ b/src/libexpr/eval-gc.hh @@ -15,9 +15,11 @@ void initGC(); */ void assertGCInitialized(); +#ifdef HAVE_BOEHMGC /** * The number of GC cycles since initGC(). */ size_t getGCCycles(); +#endif -} +} // namespace nix diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc index eb5761638..2846eccbc 100644 --- a/src/libexpr/eval-settings.cc +++ b/src/libexpr/eval-settings.cc @@ -8,7 +8,7 @@ namespace nix { /* Very hacky way to parse $NIX_PATH, which is colon-separated, but can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */ -static Strings parseNixPath(const std::string & s) +Strings EvalSettings::parseNixPath(const std::string & s) { Strings res; @@ -48,10 +48,7 @@ EvalSettings::EvalSettings(bool & readOnlyMode, EvalSettings::LookupPathHooks lo : readOnlyMode{readOnlyMode} , lookupPathHooks{lookupPathHooks} { - auto var = getEnv("NIX_PATH"); - if (var) nixPath = parseNixPath(*var); - - var = getEnv("NIX_ABORT_ON_WARN"); + auto var = getEnv("NIX_ABORT_ON_WARN"); if (var && (var == "1" || var == "yes" || var == "true")) builtinsAbortOnWarn = true; } diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index 89a42caba..0cfc14c1b 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -47,6 +47,8 @@ struct EvalSettings : Config static bool isPseudoUrl(std::string_view s); + static Strings parseNixPath(const std::string & s); + static std::string resolvePseudoUrl(std::string_view url); LookupPathHooks lookupPathHooks; @@ -63,7 +65,7 @@ struct EvalSettings : Config extern "C" typedef void (*ValueInitialiser) (EvalState & state, Value & v); ``` - The [Nix C++ API documentation](@docroot@/contributing/documentation.md#api-documentation) has more details on evaluator internals. + The [Nix C++ API documentation](@docroot@/development/documentation.md#api-documentation) has more details on evaluator internals. - `builtins.exec` *arguments* @@ -71,25 +73,30 @@ struct EvalSettings : Config )"}; Setting nixPath{ - this, getDefaultNixPath(), "nix-path", + this, {}, "nix-path", R"( List of search paths to use for [lookup path](@docroot@/language/constructs/lookup-path.md) resolution. This setting determines the value of [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath) and can be used with [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile). - The default value is + - The configuration setting is overridden by the [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) + environment variable. + - `NIX_PATH` is overridden by [specifying the setting as the command line flag](@docroot@/command-ref/conf-file.md#command-line-flags) `--nix-path`. + - Any current value is extended by the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I) or `--extra-nix-path`. - ``` - $HOME/.nix-defexpr/channels - nixpkgs=$NIX_STATE_DIR/profiles/per-user/root/channels/nixpkgs - $NIX_STATE_DIR/profiles/per-user/root/channels - ``` + If the respective paths are accessible, the default values are: - It can be overridden with the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH) or the [`-I` command line option](@docroot@/command-ref/opt-common.md#opt-I). + - `$HOME/.nix-defexpr/channels` + - `nixpkgs=$NIX_STATE_DIR/profiles/per-user/root/channels/nixpkgs` + - `$NIX_STATE_DIR/profiles/per-user/root/channels` + + See [`NIX_STATE_DIR`](@docroot@/command-ref/env-common.md#env-NIX_STATE_DIR) for details. > **Note** > - > If [pure evaluation](#conf-pure-eval) is enabled, `nixPath` evaluates to the empty list `[ ]`. + > If [restricted evaluation](@docroot@/command-ref/conf-file.md#conf-restrict-eval) is enabled, the default value is empty. + > + > If [pure evaluation](#conf-pure-eval) is enabled, `builtins.nixPath` *always* evaluates to the empty list `[ ]`. )", {}, false}; Setting currentSystem{ diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index b192f9b4b..c9101678c 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -149,7 +149,7 @@ std::string_view showType(ValueType type, bool withArticle) case nFloat: return WA("a", "float"); case nThunk: return WA("a", "thunk"); } - abort(); + unreachable(); } @@ -215,7 +215,7 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env) static constexpr size_t BASE_ENV_SIZE = 128; EvalState::EvalState( - const LookupPath & _lookupPath, + const LookupPath & lookupPathFromArguments, ref store, const fetchers::Settings & fetchSettings, const EvalSettings & settings, @@ -331,12 +331,21 @@ EvalState::EvalState( vStringSymlink.mkString("symlink"); vStringUnknown.mkString("unknown"); - /* Initialise the Nix expression search path. */ + /* Construct the Nix expression search path. */ + assert(lookupPath.elements.empty()); if (!settings.pureEval) { - for (auto & i : _lookupPath.elements) + for (auto & i : lookupPathFromArguments.elements) { lookupPath.elements.emplace_back(LookupPath::Elem {i}); - for (auto & i : settings.nixPath.get()) + } + /* $NIX_PATH overriding regular settings is implemented as a hack in `initGC()` */ + for (auto & i : settings.nixPath.get()) { lookupPath.elements.emplace_back(LookupPath::Elem::parse(i)); + } + if (!settings.restrictEval) { + for (auto & i : EvalSettings::getDefaultNixPath()) { + lookupPath.elements.emplace_back(LookupPath::Elem::parse(i)); + } + } } /* Allow access to all paths in the search path. */ @@ -771,7 +780,7 @@ void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr & case ReplExitStatus::Continue: break; default: - abort(); + unreachable(); } } } @@ -1140,7 +1149,7 @@ inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v, const PosIdx po void Expr::eval(EvalState & state, Env & env, Value & v) { - abort(); + unreachable(); } @@ -1573,7 +1582,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & .withFrame(*fun.payload.lambda.env, lambda) .debugThrow(); } - abort(); // can't happen + unreachable(); } } @@ -1970,7 +1979,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) NixStringContext context; std::vector s; size_t sSize = 0; - NixInt n = 0; + NixInt n{0}; NixFloat nf = 0; bool first = !forceString; @@ -2014,17 +2023,22 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) if (firstType == nInt) { if (vTmp.type() == nInt) { - n += vTmp.integer(); + auto newN = n + vTmp.integer(); + if (auto checked = newN.valueChecked(); checked.has_value()) { + n = NixInt(*checked); + } else { + state.error("integer overflow in adding %1% + %2%", n, vTmp.integer()).atPos(i_pos).debugThrow(); + } } else if (vTmp.type() == nFloat) { // Upgrade the type from int to float; firstType = nFloat; - nf = n; + nf = n.value; nf += vTmp.fpoint(); } else state.error("cannot add %1% to an integer", showType(vTmp)).atPos(i_pos).withFrame(env, *this).debugThrow(); } else if (firstType == nFloat) { if (vTmp.type() == nInt) { - nf += vTmp.integer(); + nf += vTmp.integer().value; } else if (vTmp.type() == nFloat) { nf += vTmp.fpoint(); } else @@ -2149,7 +2163,7 @@ NixFloat EvalState::forceFloat(Value & v, const PosIdx pos, std::string_view err try { forceValue(v, pos); if (v.type() == nInt) - return v.integer(); + return v.integer().value; else if (v.type() != nFloat) error( "expected a float but found %1%: %2%", @@ -2336,7 +2350,7 @@ BackedStringView EvalState::coerceToString( shell scripting convenience, just like `null'. */ if (v.type() == nBool && v.boolean()) return "1"; if (v.type() == nBool && !v.boolean()) return ""; - if (v.type() == nInt) return std::to_string(v.integer()); + if (v.type() == nInt) return std::to_string(v.integer().value); if (v.type() == nFloat) return std::to_string(v.fpoint()); if (v.type() == nNull) return ""; @@ -2719,9 +2733,9 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v // Special case type-compatibility between float and int if (v1.type() == nInt && v2.type() == nFloat) - return v1.integer() == v2.fpoint(); + return v1.integer().value == v2.fpoint(); if (v1.type() == nFloat && v2.type() == nInt) - return v1.fpoint() == v2.integer(); + return v1.fpoint() == v2.integer().value; // All other types are not compatible with each other. if (v1.type() != v2.type()) return false; @@ -2851,8 +2865,10 @@ void EvalState::printStatistics() topObj["cpuTime"] = cpuTime; #endif topObj["time"] = { +#ifndef _WIN32 // TODO implement {"cpu", cpuTime}, -#ifdef HAVE_BOEHMGC +#endif +#if HAVE_BOEHMGC {GC_is_incremental_mode() ? "gcNonIncremental" : "gc", gcFullOnlyTime}, {GC_is_incremental_mode() ? "gcNonIncrementalFraction" : "gcFraction", gcFullOnlyTime / cpuTime}, #endif diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 7041a3932..20963ec91 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -246,8 +246,8 @@ NixInt PackageInfo::queryMetaInt(const std::string & name, NixInt def) if (v->type() == nString) { /* Backwards compatibility with before we had support for integer meta fields. */ - if (auto n = string2Int(v->c_str())) - return *n; + if (auto n = string2Int(v->c_str())) + return NixInt{*n}; } return def; } diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 20bee193f..9ac56541a 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -2,6 +2,7 @@ #include "value.hh" #include "eval.hh" +#include #include #include @@ -42,7 +43,7 @@ class JSONSax : nlohmann::json_sax { auto attrs2 = state.buildBindings(attrs.size()); for (auto & i : attrs) attrs2.insert(i.first, i.second); - parent->value(state).mkAttrs(attrs2.alreadySorted()); + parent->value(state).mkAttrs(attrs2); return std::move(parent); } void add() override { v = nullptr; } @@ -80,42 +81,46 @@ class JSONSax : nlohmann::json_sax { public: JSONSax(EvalState & state, Value & v) : state(state), rs(new JSONState(&v)) {}; - bool null() + bool null() override { rs->value(state).mkNull(); rs->add(); return true; } - bool boolean(bool val) + bool boolean(bool val) override { rs->value(state).mkBool(val); rs->add(); return true; } - bool number_integer(number_integer_t val) + bool number_integer(number_integer_t val) override { rs->value(state).mkInt(val); rs->add(); return true; } - bool number_unsigned(number_unsigned_t val) + bool number_unsigned(number_unsigned_t val_) override { + if (val_ > std::numeric_limits::max()) { + throw Error("unsigned json number %1% outside of Nix integer range", val_); + } + NixInt::Inner val = val_; rs->value(state).mkInt(val); rs->add(); return true; } - bool number_float(number_float_t val, const string_t & s) + bool number_float(number_float_t val, const string_t & s) override { rs->value(state).mkFloat(val); rs->add(); return true; } - bool string(string_t & val) + bool string(string_t & val) override { rs->value(state).mkString(val); rs->add(); @@ -123,7 +128,7 @@ public: } #if NLOHMANN_JSON_VERSION_MAJOR >= 3 && NLOHMANN_JSON_VERSION_MINOR >= 8 - bool binary(binary_t&) + bool binary(binary_t&) override { // This function ought to be unreachable assert(false); @@ -131,35 +136,35 @@ public: } #endif - bool start_object(std::size_t len) + bool start_object(std::size_t len) override { rs = std::make_unique(std::move(rs)); return true; } - bool key(string_t & name) + bool key(string_t & name) override { dynamic_cast(rs.get())->key(name, state); return true; } - bool end_object() { + bool end_object() override { rs = rs->resolve(state); rs->add(); return true; } - bool end_array() { + bool end_array() override { return end_object(); } - bool start_array(size_t len) { + bool start_array(size_t len) override { rs = std::make_unique(std::move(rs), len != std::numeric_limits::max() ? len : 128); return true; } - bool parse_error(std::size_t, const std::string&, const nlohmann::detail::exception& ex) { + bool parse_error(std::size_t, const std::string&, const nlohmann::detail::exception& ex) override { throw JSONParseError("%s", ex.what()); } }; diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 58401be8e..a7e44cb72 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -67,6 +67,14 @@ static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length) return {result, size_t(t - result)}; } +static void requireExperimentalFeature(const ExperimentalFeature & feature, const Pos & pos) +{ + if (!experimentalFeatureSettings.isEnabled(feature)) + throw ParseError(ErrorInfo{ + .msg = HintFmt("experimental Nix feature '%1%' is disabled; add '--extra-experimental-features %1%' to enable it", showExperimentalFeature(feature)), + .pos = pos, + }); +} } @@ -119,12 +127,18 @@ or { return OR_KW; } \-\> { return IMPL; } \/\/ { return UPDATE; } \+\+ { return CONCAT; } +\<\| { requireExperimentalFeature(Xp::PipeOperators, state->positions[CUR_POS]); + return PIPE_FROM; + } +\|\> { requireExperimentalFeature(Xp::PipeOperators, state->positions[CUR_POS]); + return PIPE_INTO; + } {ID} { yylval->id = {yytext, (size_t) yyleng}; return ID; } {INT} { errno = 0; std::optional numMay = string2Int(yytext); if (numMay.has_value()) { - yylval->n = *numMay; + yylval->n = NixInt{*numMay}; } else { throw ParseError(ErrorInfo{ .msg = HintFmt("invalid integer '%1%'", yytext), diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index fa90e7b41..4d8a38b43 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -32,6 +32,7 @@ subdir('build-utils-meson/threads') boost = dependency( 'boost', modules : ['container', 'context'], + include_type: 'system', ) # boost is a public dependency, but not a pkg-config dependency unfortunately, so we # put in `deps_other`. @@ -55,7 +56,12 @@ if bdw_gc.found() endif configdata.set('HAVE_BOEHMGC', bdw_gc.found().to_int()) -toml11 = dependency('toml11', version : '>=3.7.0', method : 'cmake') +toml11 = dependency( + 'toml11', + version : '>=3.7.0', + method : 'cmake', + include_type: 'system', +) deps_other += toml11 config_h = configure_file( diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 6c6769cfd..dbc74faf9 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -25,7 +25,7 @@ std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol) void Expr::show(const SymbolTable & symbols, std::ostream & str) const { - abort(); + unreachable(); } void ExprInt::show(const SymbolTable & symbols, std::ostream & str) const @@ -271,7 +271,7 @@ std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath) void Expr::bindVars(EvalState & es, const std::shared_ptr & env) { - abort(); + unreachable(); } void ExprInt::bindVars(EvalState & es, const std::shared_ptr & env) diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 1bcc962c5..7868834f1 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -107,6 +107,7 @@ struct ExprInt : Expr { Value v; ExprInt(NixInt n) { v.mkInt(n); }; + ExprInt(NixInt::Inner n) { v.mkInt(n); }; Value * maybeThunk(EvalState & state, Env & env) override; COMMON_METHODS }; @@ -186,7 +187,7 @@ struct ExprInheritFrom : ExprVar this->fromWith = nullptr; } - void bindVars(EvalState & es, const std::shared_ptr & env); + void bindVars(EvalState & es, const std::shared_ptr & env) override; }; struct ExprSelect : Expr @@ -203,7 +204,7 @@ struct ExprSelect : Expr * * @param[out] v The attribute set that should contain the last attribute name (if it exists). * @return The last attribute name in `attrPath` - * + * * @note This does *not* evaluate the final attribute, and does not fail if that's the only attribute that does not exist. */ Symbol evalExceptFinalSelect(EvalState & state, Env & env, Value & attrs); diff --git a/src/libexpr/package.nix b/src/libexpr/package.nix index 704456c96..4d10079ff 100644 --- a/src/libexpr/package.nix +++ b/src/libexpr/package.nix @@ -102,12 +102,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libexpr/parser-state.hh b/src/libexpr/parser-state.hh index 4bb5c9204..8ad0d9ad7 100644 --- a/src/libexpr/parser-state.hh +++ b/src/libexpr/parser-state.hh @@ -20,6 +20,7 @@ struct StringToken operator std::string_view() const { return {p, l}; } }; +// This type must be trivially copyable; see YYLTYPE_IS_TRIVIAL in parser.y. struct ParserLocation { int beginOffset; @@ -86,7 +87,7 @@ struct ParserState void dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos); void dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos); - void addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * e, const PosIdx pos); + void addAttr(ExprAttrs * attrs, AttrPath && attrPath, const ParserLocation & loc, Expr * e, const ParserLocation & exprLoc); Formals * validateFormals(Formals * formals, PosIdx pos = noPos, Symbol arg = {}); Expr * stripIndentation(const PosIdx pos, std::vector>> && es); @@ -110,11 +111,12 @@ inline void ParserState::dupAttr(Symbol attr, const PosIdx pos, const PosIdx pre }); } -inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * e, const PosIdx pos) +inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, const ParserLocation & loc, Expr * e, const ParserLocation & exprLoc) { AttrPath::iterator i; // All attrpaths have at least one attr assert(!attrPath.empty()); + auto pos = at(loc); // Checking attrPath validity. // =========================== for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { @@ -179,6 +181,12 @@ inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * } else { attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos)); } + + auto it = lexerState.positionToDocComment.find(pos); + if (it != lexerState.positionToDocComment.end()) { + e->setDocComment(it->second); + lexerState.positionToDocComment.emplace(at(exprLoc), it->second); + } } inline Formals * ParserState::validateFormals(Formals * formals, PosIdx pos, Symbol arg) diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 32da2dfff..1ee2a2e41 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -1,4 +1,4 @@ -%glr-parser +%define api.location.type { ::nix::ParserLocation } %define api.pure %locations %define parse.error verbose @@ -8,8 +8,7 @@ %parse-param { nix::ParserState * state } %lex-param { void * scanner } %lex-param { nix::ParserState * state } -%expect 1 -%expect-rr 1 +%expect 0 %code requires { @@ -27,7 +26,17 @@ #include "eval-settings.hh" #include "parser-state.hh" -#define YYLTYPE ::nix::ParserLocation +// Bison seems to have difficulty growing the parser stack when using C++ with +// a custom location type. This undocumented macro tells Bison that our +// location type is "trivially copyable" in C++-ese, so it is safe to use the +// same memcpy macro it uses to grow the stack that it uses with its own +// default location type. Without this, we get "error: memory exhausted" when +// parsing some large Nix files. Our other options are to increase the initial +// stack size (200 by default) to be as large as we ever want to support (so +// that growing the stack is unnecessary), or redefine the stack-relocation +// macro ourselves (which is also undocumented). +#define YYLTYPE_IS_TRIVIAL 1 + #define YY_DECL int yylex \ (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParserState * state) @@ -77,7 +86,7 @@ YY_DECL; using namespace nix; -#define CUR_POS state->at(*yylocp) +#define CUR_POS state->at(yyloc) void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * error) @@ -99,6 +108,14 @@ static void setDocPosition(const LexerState & lexerState, ExprLambda * lambda, P } } +static Expr * makeCall(PosIdx pos, Expr * fn, Expr * arg) { + if (auto e2 = dynamic_cast(fn)) { + e2->args.push_back(arg); + return fn; + } + return new ExprCall(pos, fn, {arg}); +} + %} @@ -123,9 +140,10 @@ static void setDocPosition(const LexerState & lexerState, ExprLambda * lambda, P %type start expr expr_function expr_if expr_op %type expr_select expr_simple expr_app +%type expr_pipe_from expr_pipe_into %type expr_list -%type binds -%type formals +%type binds binds1 +%type formals formal_set %type formal %type attrpath %type attrs @@ -140,6 +158,7 @@ static void setDocPosition(const LexerState & lexerState, ExprLambda * lambda, P %token PATH HPATH SPATH PATH_END %token URI %token IF THEN ELSE ASSERT WITH LET IN_KW REC INHERIT EQ NEQ AND OR IMPL OR_KW +%token PIPE_FROM PIPE_INTO /* <| and |> */ %token DOLLAR_CURLY /* == ${ */ %token IND_STRING_OPEN IND_STRING_CLOSE %token ELLIPSIS @@ -170,22 +189,22 @@ expr_function $$ = me; SET_DOC_POS(me, @1); } - | '{' formals '}' ':' expr_function - { auto me = new ExprLambda(CUR_POS, state->validateFormals($2), $5); + | formal_set ':' expr_function[body] + { auto me = new ExprLambda(CUR_POS, state->validateFormals($formal_set), $body); $$ = me; SET_DOC_POS(me, @1); } - | '{' formals '}' '@' ID ':' expr_function + | formal_set '@' ID ':' expr_function[body] { - auto arg = state->symbols.create($5); - auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($2, CUR_POS, arg), $7); + auto arg = state->symbols.create($ID); + auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($formal_set, CUR_POS, arg), $body); $$ = me; SET_DOC_POS(me, @1); } - | ID '@' '{' formals '}' ':' expr_function + | ID '@' formal_set ':' expr_function[body] { - auto arg = state->symbols.create($1); - auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($4, CUR_POS, arg), $7); + auto arg = state->symbols.create($ID); + auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($formal_set, CUR_POS, arg), $body); $$ = me; SET_DOC_POS(me, @1); } @@ -206,9 +225,21 @@ expr_function expr_if : IF expr THEN expr ELSE expr { $$ = new ExprIf(CUR_POS, $2, $4, $6); } + | expr_pipe_from + | expr_pipe_into | expr_op ; +expr_pipe_from + : expr_op PIPE_FROM expr_pipe_from { $$ = makeCall(state->at(@2), $1, $3); } + | expr_op PIPE_FROM expr_op { $$ = makeCall(state->at(@2), $1, $3); } + ; + +expr_pipe_into + : expr_pipe_into PIPE_INTO expr_op { $$ = makeCall(state->at(@2), $3, $1); } + | expr_op PIPE_INTO expr_op { $$ = makeCall(state->at(@2), $3, $1); } + ; + expr_op : '!' expr_op %prec NOT { $$ = new ExprOpNot($2); } | '-' expr_op %prec NEGATE { $$ = new ExprCall(CUR_POS, new ExprVar(state->s.sub), {new ExprInt(0), $2}); } @@ -234,13 +265,7 @@ expr_op ; expr_app - : expr_app expr_select { - if (auto e2 = dynamic_cast($1)) { - e2->args.push_back($2); - $$ = $1; - } else - $$ = new ExprCall(CUR_POS, $1, {$2}); - } + : expr_app expr_select { $$ = makeCall(CUR_POS, $1, $2); } | expr_select ; @@ -296,11 +321,13 @@ expr_simple /* Let expressions `let {..., body = ...}' are just desugared into `(rec {..., body = ...}).body'. */ | LET '{' binds '}' - { $3->recursive = true; $$ = new ExprSelect(noPos, $3, state->s.body); } + { $3->recursive = true; $3->pos = CUR_POS; $$ = new ExprSelect(noPos, $3, state->s.body); } | REC '{' binds '}' - { $3->recursive = true; $$ = $3; } - | '{' binds '}' - { $$ = $2; } + { $3->recursive = true; $3->pos = CUR_POS; $$ = $3; } + | '{' binds1 '}' + { $2->pos = CUR_POS; $$ = $2; } + | '{' '}' + { $$ = new ExprAttrs(CUR_POS); } | '[' expr_list ']' { $$ = $2; } ; @@ -349,52 +376,50 @@ ind_string_parts ; binds - : binds attrpath '=' expr ';' { - $$ = $1; + : binds1 + | { $$ = new ExprAttrs; } + ; - auto pos = state->at(@2); - auto exprPos = state->at(@4); - { - auto it = state->lexerState.positionToDocComment.find(pos); - if (it != state->lexerState.positionToDocComment.end()) { - $4->setDocComment(it->second); - state->lexerState.positionToDocComment.emplace(exprPos, it->second); - } - } - - state->addAttr($$, std::move(*$2), $4, pos); - delete $2; +binds1 + : binds1[accum] attrpath '=' expr ';' + { $$ = $accum; + state->addAttr($$, std::move(*$attrpath), @attrpath, $expr, @expr); + delete $attrpath; } - | binds INHERIT attrs ';' - { $$ = $1; - for (auto & [i, iPos] : *$3) { - if ($$->attrs.find(i.symbol) != $$->attrs.end()) - state->dupAttr(i.symbol, iPos, $$->attrs[i.symbol].pos); - $$->attrs.emplace( + | binds[accum] INHERIT attrs ';' + { $$ = $accum; + for (auto & [i, iPos] : *$attrs) { + if ($accum->attrs.find(i.symbol) != $accum->attrs.end()) + state->dupAttr(i.symbol, iPos, $accum->attrs[i.symbol].pos); + $accum->attrs.emplace( i.symbol, ExprAttrs::AttrDef(new ExprVar(iPos, i.symbol), iPos, ExprAttrs::AttrDef::Kind::Inherited)); } - delete $3; + delete $attrs; } - | binds INHERIT '(' expr ')' attrs ';' - { $$ = $1; - if (!$$->inheritFromExprs) - $$->inheritFromExprs = std::make_unique>(); - $$->inheritFromExprs->push_back($4); - auto from = new nix::ExprInheritFrom(state->at(@4), $$->inheritFromExprs->size() - 1); - for (auto & [i, iPos] : *$6) { - if ($$->attrs.find(i.symbol) != $$->attrs.end()) - state->dupAttr(i.symbol, iPos, $$->attrs[i.symbol].pos); - $$->attrs.emplace( + | binds[accum] INHERIT '(' expr ')' attrs ';' + { $$ = $accum; + if (!$accum->inheritFromExprs) + $accum->inheritFromExprs = std::make_unique>(); + $accum->inheritFromExprs->push_back($expr); + auto from = new nix::ExprInheritFrom(state->at(@expr), $accum->inheritFromExprs->size() - 1); + for (auto & [i, iPos] : *$attrs) { + if ($accum->attrs.find(i.symbol) != $accum->attrs.end()) + state->dupAttr(i.symbol, iPos, $accum->attrs[i.symbol].pos); + $accum->attrs.emplace( i.symbol, ExprAttrs::AttrDef( new ExprSelect(iPos, from, i.symbol), iPos, ExprAttrs::AttrDef::Kind::InheritedFrom)); } - delete $6; + delete $attrs; + } + | attrpath '=' expr ';' + { $$ = new ExprAttrs; + state->addAttr($$, std::move(*$attrpath), @attrpath, $expr, @expr); + delete $attrpath; } - | { $$ = new ExprAttrs(state->at(@0)); } ; attrs @@ -452,15 +477,19 @@ expr_list | { $$ = new ExprList; } ; +formal_set + : '{' formals ',' ELLIPSIS '}' { $$ = $formals; $$->ellipsis = true; } + | '{' ELLIPSIS '}' { $$ = new Formals; $$->ellipsis = true; } + | '{' formals ',' '}' { $$ = $formals; $$->ellipsis = false; } + | '{' formals '}' { $$ = $formals; $$->ellipsis = false; } + | '{' '}' { $$ = new Formals; $$->ellipsis = false; } + ; + formals - : formal ',' formals - { $$ = $3; $$->formals.emplace_back(*$1); delete $1; } + : formals[accum] ',' formal + { $$ = $accum; $$->formals.emplace_back(*$formal); delete $formal; } | formal - { $$ = new Formals; $$->formals.emplace_back(*$1); $$->ellipsis = false; delete $1; } - | - { $$ = new Formals; $$->ellipsis = false; } - | ELLIPSIS - { $$ = new Formals; $$->ellipsis = true; } + { $$ = new Formals; $$->formals.emplace_back(*$formal); delete $formal; } ; formal diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 270e97eef..ffc8f1f63 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -426,7 +426,7 @@ static void prim_typeOf(EvalState & state, const PosIdx pos, Value * * args, Val t = args[0]->external()->typeOf(); break; case nFloat: t = "float"; break; - case nThunk: abort(); + case nThunk: unreachable(); } v.mkString(t); } @@ -587,9 +587,9 @@ struct CompareValues { try { if (v1->type() == nFloat && v2->type() == nInt) - return v1->fpoint() < v2->integer(); + return v1->fpoint() < v2->integer().value; if (v1->type() == nInt && v2->type() == nFloat) - return v1->integer() < v2->fpoint(); + return v1->integer().value < v2->fpoint(); if (v1->type() != v2->type()) state.error("cannot compare %s with %s", showType(*v1), showType(*v2)).debugThrow(); // Allow selecting a subset of enum values @@ -1843,34 +1843,6 @@ static RegisterPrimOp primop_findFile(PrimOp { .doc = R"( Find *lookup-path* in *search-path*. - A search path is represented list of [attribute sets](./types.md#attribute-set) with two attributes: - - `prefix` is a relative path. - - `path` denotes a file system location - The exact syntax depends on the command line interface. - - Examples of search path attribute sets: - - - ``` - { - prefix = "nixos-config"; - path = "/etc/nixos/configuration.nix"; - } - ``` - - - ``` - { - prefix = ""; - path = "/nix/var/nix/profiles/per-user/root/channels"; - } - ``` - - The lookup algorithm checks each entry until a match is found, returning a [path value](@docroot@/language/types.md#type-path) of the match: - - - If *lookup-path* matches `prefix`, then the remainder of *lookup-path* (the "suffix") is searched for within the directory denoted by `path`. - Note that the `path` may need to be downloaded at this point to look inside. - - If the suffix is found inside that directory, then the entry is a match. - The combined absolute path of the directory (now downloaded if need be) and the suffix is returned. - [Lookup path](@docroot@/language/constructs/lookup-path.md) expressions are [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.nixPath`](#builtins-nixPath): ```nix @@ -1882,6 +1854,119 @@ static RegisterPrimOp primop_findFile(PrimOp { ```nix builtins.findFile builtins.nixPath "nixpkgs" ``` + + A search path is represented as a list of [attribute sets](./types.md#attribute-set) with two attributes: + - `prefix` is a relative path. + - `path` denotes a file system location + + Examples of search path attribute sets: + + - ``` + { + prefix = ""; + path = "/nix/var/nix/profiles/per-user/root/channels"; + } + ``` + - ``` + { + prefix = "nixos-config"; + path = "/etc/nixos/configuration.nix"; + } + ``` + - ``` + { + prefix = "nixpkgs"; + path = "https://github.com/NixOS/nixpkgs/tarballs/master"; + } + ``` + - ``` + { + prefix = "nixpkgs"; + path = "channel:nixpkgs-unstable"; + } + ``` + - ``` + { + prefix = "flake-compat"; + path = "flake:github:edolstra/flake-compat"; + } + ``` + + The lookup algorithm checks each entry until a match is found, returning a [path value](@docroot@/language/types.md#type-path) of the match: + + - If a prefix of `lookup-path` matches `prefix`, then the remainder of *lookup-path* (the "suffix") is searched for within the directory denoted by `path`. + The contents of `path` may need to be downloaded at this point to look inside. + + - If the suffix is found inside that directory, then the entry is a match. + The combined absolute path of the directory (now downloaded if need be) and the suffix is returned. + + > **Example** + > + > A *search-path* value + > + > ``` + > [ + > { + > prefix = ""; + > path = "/home/eelco/Dev"; + > } + > { + > prefix = "nixos-config"; + > path = "/etc/nixos"; + > } + > ] + > ``` + > + > and a *lookup-path* value `"nixos-config"` will cause Nix to try `/home/eelco/Dev/nixos-config` and `/etc/nixos` in that order and return the first path that exists. + + If `path` starts with `http://` or `https://`, it is interpreted as the URL of a tarball that will be downloaded and unpacked to a temporary location. + The tarball must consist of a single top-level directory. + + The URLs of the tarballs from the official `nixos.org` channels can be abbreviated as `channel:`. + See [documentation on `nix-channel`](@docroot@/command-ref/nix-channel.md) for details about channels. + + > **Example** + > + > These two search path entries are equivalent: + > + > - ``` + > { + > prefix = "nixpkgs"; + > path = "channel:nixpkgs-unstable"; + > } + > ``` + > - ``` + > { + > prefix = "nixpkgs"; + > path = "https://nixos.org/channels/nixos-unstable/nixexprs.tar.xz"; + > } + > ``` + + Search paths can also point to source trees using [flake URLs](@docroot@/command-ref/new-cli/nix3-flake.md#url-like-syntax). + + + > **Example** + > + > The search path entry + > + > ``` + > { + > prefix = "nixpkgs"; + > path = "flake:nixpkgs"; + > } + > ``` + > specifies that the prefix `nixpkgs` shall refer to the source tree downloaded from the `nixpkgs` entry in the flake registry. + > + > Similarly + > + > ``` + > { + > prefix = "nixpkgs"; + > path = "flake:github:nixos/nixpkgs/nixos-22.05"; + > } + > ``` + > + > makes `` refer to a particular branch of the `NixOS/nixpkgs` repository on GitHub. )", .fun = prim_findFile, }); @@ -2677,13 +2762,13 @@ static struct LazyPosAcessors { PrimOp primop_lineOfPos{ .arity = 1, .fun = [] (EvalState & state, PosIdx pos, Value * * args, Value & v) { - v.mkInt(state.positions[PosIdx(args[0]->integer())].line); + v.mkInt(state.positions[PosIdx(args[0]->integer().value)].line); } }; PrimOp primop_columnOfPos{ .arity = 1, .fun = [] (EvalState & state, PosIdx pos, Value * * args, Value & v) { - v.mkInt(state.positions[PosIdx(args[0]->integer())].column); + v.mkInt(state.positions[PosIdx(args[0]->integer().value)].column); } }; @@ -3159,7 +3244,8 @@ static void elemAt(EvalState & state, const PosIdx pos, Value & list, int n, Val /* Return the n-1'th element of a list. */ static void prim_elemAt(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - elemAt(state, pos, *args[0], state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.elemAt"), v); + NixInt::Inner elem = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.elemAt").value; + elemAt(state, pos, *args[0], elem, v); } static RegisterPrimOp primop_elemAt({ @@ -3453,10 +3539,12 @@ static RegisterPrimOp primop_all({ static void prim_genList(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto len = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.genList"); + auto len_ = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.genList").value; - if (len < 0) - state.error("cannot create list of size %1%", len).atPos(pos).debugThrow(); + if (len_ < 0) + state.error("cannot create list of size %1%", len_).atPos(pos).debugThrow(); + + size_t len = size_t(len_); // More strict than striclty (!) necessary, but acceptable // as evaluating map without accessing any values makes little sense. @@ -3713,9 +3801,17 @@ static void prim_add(EvalState & state, const PosIdx pos, Value * * args, Value if (args[0]->type() == nFloat || args[1]->type() == nFloat) v.mkFloat(state.forceFloat(*args[0], pos, "while evaluating the first argument of the addition") + state.forceFloat(*args[1], pos, "while evaluating the second argument of the addition")); - else - v.mkInt( state.forceInt(*args[0], pos, "while evaluating the first argument of the addition") - + state.forceInt(*args[1], pos, "while evaluating the second argument of the addition")); + else { + auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument of the addition"); + auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument of the addition"); + + auto result_ = i1 + i2; + if (auto result = result_.valueChecked(); result.has_value()) { + v.mkInt(*result); + } else { + state.error("integer overflow in adding %1% + %2%", i1, i2).atPos(pos).debugThrow(); + } + } } static RegisterPrimOp primop_add({ @@ -3734,9 +3830,18 @@ static void prim_sub(EvalState & state, const PosIdx pos, Value * * args, Value if (args[0]->type() == nFloat || args[1]->type() == nFloat) v.mkFloat(state.forceFloat(*args[0], pos, "while evaluating the first argument of the subtraction") - state.forceFloat(*args[1], pos, "while evaluating the second argument of the subtraction")); - else - v.mkInt( state.forceInt(*args[0], pos, "while evaluating the first argument of the subtraction") - - state.forceInt(*args[1], pos, "while evaluating the second argument of the subtraction")); + else { + auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument of the subtraction"); + auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument of the subtraction"); + + auto result_ = i1 - i2; + + if (auto result = result_.valueChecked(); result.has_value()) { + v.mkInt(*result); + } else { + state.error("integer overflow in subtracting %1% - %2%", i1, i2).atPos(pos).debugThrow(); + } + } } static RegisterPrimOp primop_sub({ @@ -3755,9 +3860,18 @@ static void prim_mul(EvalState & state, const PosIdx pos, Value * * args, Value if (args[0]->type() == nFloat || args[1]->type() == nFloat) v.mkFloat(state.forceFloat(*args[0], pos, "while evaluating the first of the multiplication") * state.forceFloat(*args[1], pos, "while evaluating the second argument of the multiplication")); - else - v.mkInt( state.forceInt(*args[0], pos, "while evaluating the first argument of the multiplication") - * state.forceInt(*args[1], pos, "while evaluating the second argument of the multiplication")); + else { + auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument of the multiplication"); + auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument of the multiplication"); + + auto result_ = i1 * i2; + + if (auto result = result_.valueChecked(); result.has_value()) { + v.mkInt(*result); + } else { + state.error("integer overflow in multiplying %1% * %2%", i1, i2).atPos(pos).debugThrow(); + } + } } static RegisterPrimOp primop_mul({ @@ -3784,10 +3898,12 @@ static void prim_div(EvalState & state, const PosIdx pos, Value * * args, Value NixInt i1 = state.forceInt(*args[0], pos, "while evaluating the first operand of the division"); NixInt i2 = state.forceInt(*args[1], pos, "while evaluating the second operand of the division"); /* Avoid division overflow as it might raise SIGFPE. */ - if (i1 == std::numeric_limits::min() && i2 == -1) - state.error("overflow in integer division").atPos(pos).debugThrow(); - - v.mkInt(i1 / i2); + auto result_ = i1 / i2; + if (auto result = result_.valueChecked(); result.has_value()) { + v.mkInt(*result); + } else { + state.error("integer overflow in dividing %1% / %2%", i1, i2).atPos(pos).debugThrow(); + } } } @@ -3802,8 +3918,9 @@ static RegisterPrimOp primop_div({ static void prim_bitAnd(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - v.mkInt(state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitAnd") - & state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitAnd")); + auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitAnd"); + auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitAnd"); + v.mkInt(i1.value & i2.value); } static RegisterPrimOp primop_bitAnd({ @@ -3817,8 +3934,10 @@ static RegisterPrimOp primop_bitAnd({ static void prim_bitOr(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - v.mkInt(state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitOr") - | state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitOr")); + auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitOr"); + auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitOr"); + + v.mkInt(i1.value | i2.value); } static RegisterPrimOp primop_bitOr({ @@ -3832,8 +3951,10 @@ static RegisterPrimOp primop_bitOr({ static void prim_bitXor(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - v.mkInt(state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitXor") - ^ state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitXor")); + auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitXor"); + auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitXor"); + + v.mkInt(i1.value ^ i2.value); } static RegisterPrimOp primop_bitXor({ @@ -3913,13 +4034,19 @@ static RegisterPrimOp primop_toString({ non-negative. */ static void prim_substring(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - int start = state.forceInt(*args[0], pos, "while evaluating the first argument (the start offset) passed to builtins.substring"); + NixInt::Inner start = state.forceInt(*args[0], pos, "while evaluating the first argument (the start offset) passed to builtins.substring").value; if (start < 0) state.error("negative start position in 'substring'").atPos(pos).debugThrow(); - int len = state.forceInt(*args[1], pos, "while evaluating the second argument (the substring length) passed to builtins.substring"); + NixInt::Inner len = state.forceInt(*args[1], pos, "while evaluating the second argument (the substring length) passed to builtins.substring").value; + + // Negative length may be idiomatically passed to builtins.substring to get + // the tail of the string. + if (len < 0) { + len = std::numeric_limits::max(); + } // Special-case on empty substring to avoid O(n) strlen // This allows for the use of empty substrings to efficently capture string context @@ -3962,7 +4089,7 @@ static void prim_stringLength(EvalState & state, const PosIdx pos, Value * * arg { NixStringContext context; auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.stringLength"); - v.mkInt(s->size()); + v.mkInt(NixInt::Inner(s->size())); } static RegisterPrimOp primop_stringLength({ @@ -4446,7 +4573,8 @@ static void prim_compareVersions(EvalState & state, const PosIdx pos, Value * * { auto version1 = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.compareVersions"); auto version2 = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.compareVersions"); - v.mkInt(compareVersions(version1, version2)); + auto result = compareVersions(version1, version2); + v.mkInt(result < 0 ? -1 : result > 0 ? 1 : 0); } static RegisterPrimOp primop_compareVersions({ @@ -4737,7 +4865,17 @@ void EvalState::createBaseEnv() addConstant("__nixPath", v, { .type = nList, .doc = R"( - The value of the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path): a list of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). + A list of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). + Its value is primarily determined by the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path), which are + - Overridden by the [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) environment variable or the `--nix-path` option + - Extended by the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I) or `--extra-nix-path` + + > **Example** + > + > ```bash + > $ NIX_PATH= nix-instantiate --eval --expr "builtins.nixPath" -I foo=bar --no-pure-eval + > [ { path = "bar"; prefix = "foo"; } ] + > ``` Lookup path expressions are [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.findFile`](./builtins.html#builtins-findFile): diff --git a/src/libexpr/primops/derivation.nix b/src/libexpr/primops/derivation.nix index c0fbe8082..f329ff71e 100644 --- a/src/libexpr/primops/derivation.nix +++ b/src/libexpr/primops/derivation.nix @@ -1,6 +1,31 @@ -/* This is the implementation of the ‘derivation’ builtin function. - It's actually a wrapper around the ‘derivationStrict’ primop. */ +# This is the implementation of the ‘derivation’ builtin function. +# It's actually a wrapper around the ‘derivationStrict’ primop. +# Note that the following comment will be shown in :doc in the repl, but not in the manual. +/** + Create a derivation. + + # Inputs + + The single argument is an attribute set that describes what to build and how to build it. + See https://nix.dev/manual/nix/2.23/language/derivations + + # Output + + The result is an attribute set that describes the derivation. + Notably it contains the outputs, which in the context of the Nix language are special strings that refer to the output paths, which may not yet exist. + The realisation of these outputs only occurs when needed; for example + + * When `nix-build` or a similar command is run, it realises the outputs that were requested on its command line. + See https://nix.dev/manual/nix/2.23/command-ref/nix-build + + * When `import`, `readFile`, `readDir` or some other functions are called, they have to realise the outputs they depend on. + This is referred to as "import from derivation". + See https://nix.dev/manual/nix/2.23/language/import-from-derivation + + Note that `derivation` is very bare-bones, and provides almost no commands during the build. + Most likely, you'll want to use functions like `stdenv.mkDerivation` in Nixpkgs to set up a basic environment. +*/ drvAttrs @ { outputs ? [ "out" ], ... }: let diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 333e486fd..78328701d 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -122,9 +122,15 @@ static void fetchTree( } else if (attr.value->type() == nBool) attrs.emplace(state.symbols[attr.name], Explicit{attr.value->boolean()}); - else if (attr.value->type() == nInt) - attrs.emplace(state.symbols[attr.name], uint64_t(attr.value->integer())); - else if (state.symbols[attr.name] == "publicKeys") { + else if (attr.value->type() == nInt) { + auto intValue = attr.value->integer().value; + + if (intValue < 0) { + state.error("negative value given for fetchTree attr %1%: %2%", state.symbols[attr.name], intValue).atPos(pos).debugThrow(); + } + + attrs.emplace(state.symbols[attr.name], uint64_t(intValue)); + } else if (state.symbols[attr.name] == "publicKeys") { experimentalFeatureSettings.require(Xp::VerifiedFetches); attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, context).dump()); } @@ -383,7 +389,7 @@ static RegisterPrimOp primop_fetchTree({ - `"mercurial"` *input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references). - The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled. + The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/development/experimental-features.md#xp-feature-flakes) to be enabled. > **Example** > @@ -559,11 +565,11 @@ static RegisterPrimOp primop_fetchTarball({ .doc = R"( Download the specified URL, unpack it and return the path of the unpacked tree. The file must be a tape archive (`.tar`) compressed - with `gzip`, `bzip2` or `xz`. The top-level path component of the - files in the tarball is removed, so it is best if the tarball - contains a single directory at top level. The typical use of the - function is to obtain external Nix expression dependencies, such as - a particular version of Nixpkgs, e.g. + with `gzip`, `bzip2` or `xz`. If the tarball consists of a + single directory, then the top-level path component of the files + in the tarball is removed. The typical use of the function is to + obtain external Nix expression dependencies, such as a + particular version of Nixpkgs, e.g. ```nix with import (fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz) {}; @@ -670,12 +676,12 @@ static RegisterPrimOp primop_fetchGit({ Whether to check `rev` for a signature matching `publicKey` or `publicKeys`. If `verifyCommit` is enabled, then `fetchGit` cannot use a local repository with uncommitted changes. - Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + Requires the [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). - `publicKey` The public key against which `rev` is verified if `verifyCommit` is enabled. - Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + Requires the [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). - `keytype` (default: `"ssh-ed25519"`) @@ -687,7 +693,7 @@ static RegisterPrimOp primop_fetchGit({ - `"ssh-ed25519"` - `"ssh-ed25519-sk"` - `"ssh-rsa"` - Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + Requires the [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). - `publicKeys` @@ -701,7 +707,7 @@ static RegisterPrimOp primop_fetchGit({ } ``` - Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + Requires the [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). Here are some examples of how to use `fetchGit`. diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 6c7d303e8..b4f1df7a8 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -2,6 +2,7 @@ #include "eval-inline.hh" #include + #include namespace nix { diff --git a/src/libexpr/print-ambiguous.cc b/src/libexpr/print-ambiguous.cc index 5d55b45da..a40c98643 100644 --- a/src/libexpr/print-ambiguous.cc +++ b/src/libexpr/print-ambiguous.cc @@ -94,7 +94,7 @@ void printAmbiguous( break; default: printError("Nix evaluator internal error: printAmbiguous: invalid value type"); - abort(); + unreachable(); } } diff --git a/src/libexpr/print.cc b/src/libexpr/print.cc index bc17d6bfe..4d1a6868c 100644 --- a/src/libexpr/print.cc +++ b/src/libexpr/print.cc @@ -475,7 +475,7 @@ private: else output << "primop"; } else { - abort(); + unreachable(); } output << "»"; @@ -504,7 +504,7 @@ private: if (options.ansiColors) output << ANSI_NORMAL; } else { - abort(); + unreachable(); } } diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh index c7a3563b0..be12f6248 100644 --- a/src/libexpr/symbol-table.hh +++ b/src/libexpr/symbol-table.hh @@ -7,6 +7,7 @@ #include "types.hh" #include "chunked-vector.hh" +#include "error.hh" namespace nix { @@ -69,6 +70,8 @@ public: auto operator<=>(const Symbol other) const { return id <=> other.id; } bool operator==(const Symbol other) const { return id == other.id; } + + friend class std::hash; }; /** @@ -113,7 +116,7 @@ public: SymbolStr operator[](Symbol s) const { if (s.id == 0 || s.id > store.size()) - abort(); + unreachable(); return SymbolStr(store[s.id - 1]); } @@ -132,3 +135,12 @@ public: }; } + +template<> +struct std::hash +{ + std::size_t operator()(const nix::Symbol & s) const noexcept + { + return std::hash{}(s.id); + } +}; diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index f8cc05616..8044fe347 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -22,7 +22,7 @@ json printValueAsJSON(EvalState & state, bool strict, switch (v.type()) { case nInt: - out = v.integer(); + out = v.integer().value; break; case nBool: diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 1f4d72d39..f68befe0e 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -8,6 +8,7 @@ #include "value/context.hh" #include "source-path.hh" #include "print-options.hh" +#include "checked-arithmetic.hh" #if HAVE_BOEHMGC #include @@ -73,8 +74,8 @@ class EvalState; class XMLWriter; class Printer; -typedef int64_t NixInt; -typedef double NixFloat; +using NixInt = checked::Checked; +using NixFloat = double; /** * External values must descend from ExternalValueBase, so that @@ -285,7 +286,7 @@ public: if (invalidIsThunk) return nThunk; else - abort(); + unreachable(); } inline void finishValue(InternalType newType, Payload newPayload) @@ -304,6 +305,11 @@ public: return internalType != tUninitialized; } + inline void mkInt(NixInt::Inner n) + { + mkInt(NixInt{n}); + } + inline void mkInt(NixInt n) { finishValue(tInt, { .integer = n }); @@ -325,9 +331,9 @@ public: void mkStringMove(const char * s, const NixStringContext & context); - inline void mkString(const Symbol & s) + inline void mkString(const SymbolStr & s) { - mkString(((const std::string &) s).c_str()); + mkString(s.c_str()); } void mkPath(const SourcePath & path); @@ -494,11 +500,11 @@ void Value::mkBlackhole() #if HAVE_BOEHMGC typedef std::vector> ValueVector; -typedef std::map, traceable_allocator>> ValueMap; +typedef std::unordered_map, std::equal_to, traceable_allocator>> ValueMap; typedef std::map, traceable_allocator>> ValueVectorMap; #else typedef std::vector ValueVector; -typedef std::map ValueMap; +typedef std::unordered_map ValueMap; typedef std::map ValueVectorMap; #endif diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index b788c5948..25d04cdc9 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -33,7 +33,7 @@ nlohmann::json attrsToJSON(const Attrs & attrs) json[attr.first] = *v; } else if (auto v = std::get_if>(&attr.second)) { json[attr.first] = v->t; - } else abort(); + } else unreachable(); } return json; } @@ -99,7 +99,7 @@ std::map attrsToQuery(const Attrs & attrs) query.insert_or_assign(attr.first, *v); } else if (auto v = std::get_if>(&attr.second)) { query.insert_or_assign(attr.first, v->t ? "1" : "0"); - } else abort(); + } else unreachable(); } return query; } diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index ecc71ae47..114aa4ec0 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -126,16 +126,39 @@ Object lookupObject(git_repository * repo, const git_oid & oid, git_object_t typ } template -T peelObject(git_repository * repo, git_object * obj, git_object_t type) +T peelObject(git_object * obj, git_object_t type) { T obj2; if (git_object_peel((git_object * *) (typename T::pointer *) Setter(obj2), obj, type)) { auto err = git_error_last(); - throw Error("peeling Git object '%s': %s", git_object_id(obj), err->message); + throw Error("peeling Git object '%s': %s", *git_object_id(obj), err->message); } return obj2; } +template +T dupObject(typename T::pointer obj) +{ + T obj2; + if (git_object_dup((git_object * *) (typename T::pointer *) Setter(obj2), (git_object *) obj)) + throw Error("duplicating object '%s': %s", *git_object_id((git_object *) obj), git_error_last()->message); + return obj2; +} + +/** + * Peel the specified object (i.e. follow tag and commit objects) to + * either a blob or a tree. + */ +static Object peelToTreeOrBlob(git_object * obj) +{ + /* git_object_peel() doesn't handle blob objects, so handle those + specially. */ + if (git_object_type(obj) == GIT_OBJECT_BLOB) + return dupObject(obj); + else + return peelObject(obj, GIT_OBJECT_TREE); +} + struct GitRepoImpl : GitRepo, std::enable_shared_from_this { /** Location of the repository on disk. */ @@ -166,7 +189,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this std::unordered_set done; std::queue todo; - todo.push(peelObject(*this, lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT)); + todo.push(peelObject(lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT)); while (auto commit = pop(todo)) { if (!done.insert(*git_commit_id(commit->get())).second) continue; @@ -184,7 +207,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this uint64_t getLastModified(const Hash & rev) override { - auto commit = peelObject(*this, lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT); + auto commit = peelObject(lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT); return git_commit_time(commit.get()); } @@ -463,6 +486,23 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this return narHash; } + + Hash dereferenceSingletonDirectory(const Hash & oid_) override + { + auto oid = hashToOID(oid_); + + auto _tree = lookupObject(*this, oid, GIT_OBJECT_TREE); + auto tree = (const git_tree *) &*_tree; + + if (git_tree_entrycount(tree) == 1) { + auto entry = git_tree_entry_byindex(tree, 0); + auto mode = git_tree_entry_filemode(entry); + if (mode == GIT_FILEMODE_TREE) + oid = *git_tree_entry_id(entry); + } + + return toHash(oid); + } }; ref GitRepo::openRepo(const std::filesystem::path & path, bool create, bool bare) @@ -476,11 +516,11 @@ ref GitRepo::openRepo(const std::filesystem::path & path, bool create, struct GitSourceAccessor : SourceAccessor { ref repo; - Tree root; + Object root; GitSourceAccessor(ref repo_, const Hash & rev) : repo(repo_) - , root(peelObject(*repo, lookupObject(*repo, hashToOID(rev)).get(), GIT_OBJECT_TREE)) + , root(peelToTreeOrBlob(lookupObject(*repo, hashToOID(rev)).get())) { } @@ -506,7 +546,7 @@ struct GitSourceAccessor : SourceAccessor std::optional maybeLstat(const CanonPath & path) override { if (path.isRoot()) - return Stat { .type = tDirectory }; + return Stat { .type = git_object_type(root.get()) == GIT_OBJECT_TREE ? tDirectory : tRegular }; auto entry = lookup(path); if (!entry) @@ -616,10 +656,10 @@ struct GitSourceAccessor : SourceAccessor std::optional lookupTree(const CanonPath & path) { if (path.isRoot()) { - Tree tree; - if (git_tree_dup(Setter(tree), root.get())) - throw Error("duplicating directory '%s': %s", showPath(path), git_error_last()->message); - return tree; + if (git_object_type(root.get()) == GIT_OBJECT_TREE) + return dupObject((git_tree *) &*root); + else + return std::nullopt; } auto entry = lookup(path); @@ -646,10 +686,10 @@ struct GitSourceAccessor : SourceAccessor std::variant getTree(const CanonPath & path) { if (path.isRoot()) { - Tree tree; - if (git_tree_dup(Setter(tree), root.get())) - throw Error("duplicating directory '%s': %s", showPath(path), git_error_last()->message); - return tree; + if (git_object_type(root.get()) == GIT_OBJECT_TREE) + return dupObject((git_tree *) &*root); + else + throw Error("Git root object '%s' is not a directory", *git_object_id(root.get())); } auto entry = need(path); @@ -669,6 +709,9 @@ struct GitSourceAccessor : SourceAccessor Blob getBlob(const CanonPath & path, bool expectSymlink) { + if (!expectSymlink && git_object_type(root.get()) == GIT_OBJECT_BLOB) + return dupObject((git_blob *) &*root); + auto notExpected = [&]() { throw Error( @@ -782,8 +825,6 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink std::vector pendingDirs; - size_t componentsToStrip = 1; - void pushBuilder(std::string name) { git_treebuilder * b; @@ -839,9 +880,6 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink { std::span pathComponents2{pathComponents}; - if (pathComponents2.size() <= componentsToStrip) return false; - pathComponents2 = pathComponents2.subspan(componentsToStrip); - updateBuilders( isDir ? pathComponents2 @@ -964,7 +1002,8 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink git_tree_entry_filemode(entry)); } - Hash sync() override { + Hash sync() override + { updateBuilders({}); auto [oid, _name] = popBuilder(); diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh index 495916f62..915252868 100644 --- a/src/libfetchers/git-utils.hh +++ b/src/libfetchers/git-utils.hh @@ -98,6 +98,13 @@ struct GitRepo * serialisation. This is memoised on-disk. */ virtual Hash treeHashToNarHash(const Hash & treeHash) = 0; + + /** + * If the specified Git object is a directory with a single entry + * that is a directory, return the ID of that object. + * Otherwise, return the passed ID unchanged. + */ + virtual Hash dereferenceSingletonDirectory(const Hash & oid) = 0; }; ref getTarballCache(); diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 2968d2df2..2e914164a 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -254,12 +254,18 @@ struct GitArchiveInputScheme : InputScheme getFileTransfer()->download(std::move(req), sink); }); + auto act = std::make_unique(*logger, lvlInfo, actUnknown, + fmt("unpacking '%s' into the Git cache", input.to_string())); + TarArchive archive { *source }; - auto parseSink = getTarballCache()->getFileSystemObjectSink(); + auto tarballCache = getTarballCache(); + auto parseSink = tarballCache->getFileSystemObjectSink(); auto lastModified = unpackTarfileToSink(archive, *parseSink); + act.reset(); + TarballInfo tarballInfo { - .treeHash = parseSink->sync(), + .treeHash = tarballCache->dereferenceSingletonDirectory(parseSink->sync()), .lastModified = lastModified }; diff --git a/src/libfetchers/package.nix b/src/libfetchers/package.nix index b4abb144b..9b5d8bff7 100644 --- a/src/libfetchers/package.nix +++ b/src/libfetchers/package.nix @@ -67,14 +67,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated - # to work with `strictDeps`. - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 55db3eafb..457210542 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -143,6 +143,9 @@ DownloadTarballResult downloadTarball( // TODO: fall back to cached value if download fails. + auto act = std::make_unique(*logger, lvlInfo, actUnknown, + fmt("unpacking '%s' into the Git cache", url)); + AutoDelete cleanupTemp; /* Note: if the download is cached, `importTarball()` will receive @@ -164,9 +167,12 @@ DownloadTarballResult downloadTarball( TarArchive{path}; }) : TarArchive{*source}; - auto parseSink = getTarballCache()->getFileSystemObjectSink(); + auto tarballCache = getTarballCache(); + auto parseSink = tarballCache->getFileSystemObjectSink(); auto lastModified = unpackTarfileToSink(archive, *parseSink); + act.reset(); + auto res(_res->lock()); Attrs infoAttrs; @@ -177,7 +183,8 @@ DownloadTarballResult downloadTarball( infoAttrs = cached->value; } else { infoAttrs.insert_or_assign("etag", res->etag); - infoAttrs.insert_or_assign("treeHash", parseSink->sync().gitRev()); + infoAttrs.insert_or_assign("treeHash", + tarballCache->dereferenceSingletonDirectory(parseSink->sync()).gitRev()); infoAttrs.insert_or_assign("lastModified", uint64_t(lastModified)); if (res->immutableUrl) infoAttrs.insert_or_assign("immutableUrl", *res->immutableUrl); diff --git a/src/libflake/flake/flake.cc b/src/libflake/flake/flake.cc index 5b9ecffe4..f7fe4df54 100644 --- a/src/libflake/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -139,9 +139,16 @@ static FlakeInput parseFlakeInput(EvalState & state, case nBool: attrs.emplace(state.symbols[attr.name], Explicit { attr.value->boolean() }); break; - case nInt: - attrs.emplace(state.symbols[attr.name], (long unsigned int) attr.value->integer()); + case nInt: { + auto intValue = attr.value->integer().value; + + if (intValue < 0) { + state.error("negative value given for flake input attribute %1%: %2%", state.symbols[attr.name], intValue).debugThrow(); + } + + attrs.emplace(state.symbols[attr.name], uint64_t(intValue)); break; + } default: if (attr.name == state.symbols.create("publicKeys")) { experimentalFeatureSettings.require(Xp::VerifiedFetches); @@ -271,7 +278,7 @@ static Flake readFlake( else if (setting.value->type() == nInt) flake.config.settings.emplace( state.symbols[setting.name], - state.forceInt(*setting.value, setting.pos, "")); + state.forceInt(*setting.value, setting.pos, "").value); else if (setting.value->type() == nBool) flake.config.settings.emplace( state.symbols[setting.name], @@ -903,8 +910,13 @@ static void prim_flakeRefToString( for (const auto & attr : *args[0]->attrs()) { auto t = attr.value->type(); if (t == nInt) { - attrs.emplace(state.symbols[attr.name], - (uint64_t) attr.value->integer()); + auto intValue = attr.value->integer().value; + + if (intValue < 0) { + state.error("negative value given for flake ref attr %1%: %2%", state.symbols[attr.name], intValue).atPos(pos).debugThrow(); + } + + attrs.emplace(state.symbols[attr.name], uint64_t(intValue)); } else if (t == nBool) { attrs.emplace(state.symbols[attr.name], Explicit { attr.value->boolean() }); diff --git a/src/libflake/flake/nix-flake.pc.in b/src/libflake/flake/nix-flake.pc.in new file mode 100644 index 000000000..10c52f5e9 --- /dev/null +++ b/src/libflake/flake/nix-flake.pc.in @@ -0,0 +1,10 @@ +prefix=@prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Nix +Description: Nix Package Manager +Version: @PACKAGE_VERSION@ +Requires: nix-util nix-store nix-expr +Libs: -L${libdir} -lnixflake +Cflags: -I${includedir}/nix -std=c++2a diff --git a/src/libflake/local.mk b/src/libflake/local.mk index 2cceda2bf..5e604ef3a 100644 --- a/src/libflake/local.mk +++ b/src/libflake/local.mk @@ -15,3 +15,8 @@ libflake_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetcher libflake_LDFLAGS += $(THREAD_LDFLAGS) libflake_LIBS = libutil libstore libfetchers libexpr + +$(eval $(call install-file-in, $(buildprefix)$(d)/flake/nix-flake.pc, $(libdir)/pkgconfig, 0644)) + +$(foreach i, $(wildcard src/libflake/flake/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644))) diff --git a/src/libflake/package.nix b/src/libflake/package.nix index af6f5da94..851adf07e 100644 --- a/src/libflake/package.nix +++ b/src/libflake/package.nix @@ -67,14 +67,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated - # to work with `strictDeps`. - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libmain-c/package.nix b/src/libmain-c/package.nix index 478e34a85..ce6f67300 100644 --- a/src/libmain-c/package.nix +++ b/src/libmain-c/package.nix @@ -68,12 +68,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc index 9829859de..a4e0530c8 100644 --- a/src/libmain/loggers.cc +++ b/src/libmain/loggers.cc @@ -36,7 +36,7 @@ Logger * makeDefaultLogger() { return logger; } default: - abort(); + unreachable(); } } diff --git a/src/libmain/package.nix b/src/libmain/package.nix index bbd97ec3e..47513dbdc 100644 --- a/src/libmain/package.nix +++ b/src/libmain/package.nix @@ -62,14 +62,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated - # to work with `strictDeps`. - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libstore-c/package.nix b/src/libstore-c/package.nix index fc34c1bda..e4f372236 100644 --- a/src/libstore-c/package.nix +++ b/src/libstore-c/package.nix @@ -64,12 +64,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh index 807054926..8c60d0198 100644 --- a/src/libstore/build/drv-output-substitution-goal.hh +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -36,7 +36,7 @@ public: Co init() override; Co realisationFetched(std::shared_ptr outputInfo, nix::ref sub); - void timedOut(Error && ex) override { abort(); }; + void timedOut(Error && ex) override { unreachable(); }; std::string key() override; diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 162c392d0..9c6a40c84 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -400,12 +400,12 @@ public: virtual void handleChildOutput(Descriptor fd, std::string_view data) { - abort(); + unreachable(); } virtual void handleEOF(Descriptor fd) { - abort(); + unreachable(); } void trace(std::string_view s); diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 7deeb4748..0152f1808 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -145,8 +145,10 @@ Goal::Co PathSubstitutionGoal::init() /* None left. Terminate this goal and let someone else deal with it. */ - worker.failedSubstitutions++; - worker.updateProgress(); + if (substituterFailed) { + worker.failedSubstitutions++; + worker.updateProgress(); + } /* Hack: don't indicate failure if there were no substituters. In that case the calling derivation should just do a @@ -158,7 +160,7 @@ Goal::Co PathSubstitutionGoal::init() } -Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref sub, std::shared_ptr info, bool& substituterFailed) +Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref sub, std::shared_ptr info, bool & substituterFailed) { trace("all references realised"); diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh index 86e4f5423..f2cf797e5 100644 --- a/src/libstore/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -50,7 +50,7 @@ public: PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); ~PathSubstitutionGoal(); - void timedOut(Error && ex) override { abort(); }; + void timedOut(Error && ex) override { unreachable(); }; /** * We prepend "a$" to the key name to ensure substitution goals @@ -66,7 +66,7 @@ public: */ Co init() override; Co gotInfo(); - Co tryToRun(StorePath subPath, nix::ref sub, std::shared_ptr info, bool& substituterFailed); + Co tryToRun(StorePath subPath, nix::ref sub, std::shared_ptr info, bool & substituterFailed); Co finished(); /** diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 7fc41b121..ab0ba67b5 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -216,7 +216,7 @@ void Worker::childStarted(GoalPtr goal, const std::set successfulStatuses {200, 201, 204, 206, 304, 0 /* other protocol */}; + /* Get the HTTP status code, or 0 for other protocols. */ long getHTTPStatus() { @@ -373,10 +376,14 @@ struct curlFileTransfer : public FileTransfer void finish(CURLcode code) { + auto finishTime = std::chrono::steady_clock::now(); + auto httpStatus = getHTTPStatus(); - debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes", - request.verb(), request.uri, code, httpStatus, result.bodySize); + debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes, duration = %.2f s", + request.verb(), request.uri, code, httpStatus, result.bodySize, + std::chrono::duration_cast(finishTime - startTime).count() / 1000.0f + ); appendCurrentUrl(); @@ -851,8 +858,10 @@ void FileTransfer::download( buffer). We don't wait forever to prevent stalling the download thread. (Hopefully sleeping will throttle the sender.) */ - if (state->data.size() > 1024 * 1024) { + if (state->data.size() > fileTransferSettings.downloadBufferSize) { debug("download buffer is full; going to sleep"); + static bool haveWarned = false; + warnOnce(haveWarned, "download buffer is full; consider increasing the 'download-buffer-size' setting"); state.wait_for(state->request, std::chrono::seconds(10)); } diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index 1f5b4ab93..d836ab2c4 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -47,6 +47,12 @@ struct FileTransferSettings : Config Setting tries{this, 5, "download-attempts", "How often Nix will attempt to download a file before giving up."}; + + Setting downloadBufferSize{this, 64 * 1024 * 1024, "download-buffer-size", + R"( + The size of Nix's internal download buffer during `curl` transfers. If data is + not processed quickly enough to exceed the size of this buffer, downloads may stall. + )"}; }; extern FileTransferSettings fileTransferSettings; diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 54e09cb16..1494712da 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -559,7 +559,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) non-blocking flag from the server socket, so explicitly make it blocking. */ if (fcntl(fdClient.get(), F_SETFL, fcntl(fdClient.get(), F_GETFL) & ~O_NONBLOCK) == -1) - abort(); + panic("Could not set non-blocking flag on client socket"); while (true) { try { @@ -891,7 +891,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) void LocalStore::autoGC(bool sync) { -#ifdef HAVE_STATVFS +#if HAVE_STATVFS static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE"); auto getAvail = [this]() -> uint64_t { diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 60b3ee34e..52ab35b4c 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -64,7 +64,6 @@ Settings::Settings() , nixStateDir(canonPath(getEnvNonEmpty("NIX_STATE_DIR").value_or(NIX_STATE_DIR))) , nixConfDir(canonPath(getEnvNonEmpty("NIX_CONF_DIR").value_or(NIX_CONF_DIR))) , nixUserConfFiles(getUserConfigFiles()) - , nixBinDir(canonPath(getEnvNonEmpty("NIX_BIN_DIR").value_or(NIX_BIN_DIR))) , nixManDir(canonPath(NIX_MAN_DIR)) , nixDaemonSocketFile(canonPath(getEnvNonEmpty("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH))) { @@ -95,34 +94,6 @@ Settings::Settings() sandboxPaths = tokenizeString("/System/Library/Frameworks /System/Library/PrivateFrameworks /bin/sh /bin/bash /private/tmp /private/var/tmp /usr/lib"); allowedImpureHostPrefixes = tokenizeString("/System/Library /usr/lib /dev /bin/sh"); #endif - - /* Set the build hook location - - For builds we perform a self-invocation, so Nix has to be self-aware. - That is, it has to know where it is installed. We don't think it's sentient. - - Normally, nix is installed according to `nixBinDir`, which is set at compile time, - but can be overridden. This makes for a great default that works even if this - code is linked as a library into some other program whose main is not aware - that it might need to be a build remote hook. - - However, it may not have been installed at all. For example, if it's a static build, - there's a good chance that it has been moved out of its installation directory. - That makes `nixBinDir` useless. Instead, we'll query the OS for the path to the - current executable, using `getSelfExe()`. - - As a last resort, we resort to `PATH`. Hopefully we find a `nix` there that's compatible. - If you're porting Nix to a new platform, that might be good enough for a while, but - you'll want to improve `getSelfExe()` to work on your platform. - */ - std::string nixExePath = nixBinDir + "/nix"; - if (!pathExists(nixExePath)) { - nixExePath = getSelfExe().value_or("nix"); - } - buildHook = { - nixExePath, - "__build-remote", - }; } void loadConfFile(AbstractConfig & config) @@ -297,7 +268,7 @@ template<> std::string BaseSetting::to_string() const if (value == smEnabled) return "true"; else if (value == smRelaxed) return "relaxed"; else if (value == smDisabled) return "false"; - else abort(); + else unreachable(); } template<> void BaseSetting::convertToArg(Args & args, const std::string & category) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 30d7537bd..65c7bf3bc 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -84,11 +84,6 @@ public: */ std::vector nixUserConfFiles; - /** - * The directory where the main programs are stored. - */ - Path nixBinDir; - /** * The directory where the man pages are stored. */ @@ -246,7 +241,7 @@ public: )", {"build-timeout"}}; - Setting buildHook{this, {}, "build-hook", + Setting buildHook{this, {"nix", "__build-remote"}, "build-hook", R"( The path to the helper program that executes remote builds. @@ -286,7 +281,7 @@ public: For backward compatibility, `ssh://` may be omitted. The hostname may be an alias defined in `~/.ssh/config`. - 2. A comma-separated list of [Nix system types](@docroot@/contributing/hacking.md#system-type). + 2. A comma-separated list of [Nix system types](@docroot@/development/building.md#system-type). If omitted, this defaults to the local platform type. > **Example** @@ -866,13 +861,13 @@ public: - `ca-derivations` - Included by default if the [`ca-derivations` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) is enabled. + Included by default if the [`ca-derivations` experimental feature](@docroot@/development/experimental-features.md#xp-feature-ca-derivations) is enabled. This system feature is implicitly required by derivations with the [`__contentAddressed` attribute](@docroot@/language/advanced-attributes.md#adv-attr-__contentAddressed). - `recursive-nix` - Included by default if the [`recursive-nix` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-recursive-nix) is enabled. + Included by default if the [`recursive-nix` experimental feature](@docroot@/development/experimental-features.md#xp-feature-recursive-nix) is enabled. - `uid-range` @@ -1131,7 +1126,10 @@ public: )"}; Setting maxFree{ - this, std::numeric_limits::max(), "max-free", + // n.b. this is deliberately int64 max rather than uint64 max because + // this goes through the Nix language JSON parser and thus needs to be + // representable in Nix language integers. + this, std::numeric_limits::max(), "max-free", R"( When a garbage collection is triggered by the `min-free` option, it stops as soon as `max-free` bytes are available. The default is @@ -1221,7 +1219,10 @@ public: Setting warnLargePathThreshold{ this, - std::numeric_limits::max(), + // n.b. this is deliberately int64 max rather than uint64 max because + // this goes through the Nix language JSON parser and thus needs to be + // representable in Nix language integers. + std::numeric_limits::max(), "warn-large-path-threshold", R"( Warn when copying a path larger than this number of bytes to the Nix store diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 5dc8f3370..88be6a366 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -71,7 +71,6 @@ libstore_CXXFLAGS += \ -DNIX_STATE_DIR=\"$(NIX_ROOT)$(localstatedir)/nix\" \ -DNIX_LOG_DIR=\"$(NIX_ROOT)$(localstatedir)/log/nix\" \ -DNIX_CONF_DIR=\"$(NIX_ROOT)$(sysconfdir)/nix\" \ - -DNIX_BIN_DIR=\"$(NIX_ROOT)$(bindir)\" \ -DNIX_MAN_DIR=\"$(NIX_ROOT)$(mandir)\" \ -DLSOF=\"$(NIX_ROOT)$(lsof)\" diff --git a/src/libstore/meson.build b/src/libstore/meson.build index cb8110f3f..8e30845e1 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -21,7 +21,7 @@ configdata = configuration_data() # TODO rename, because it will conflict with downstream projects configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) -configdata.set_quoted('SYSTEM', host_machine.system()) +configdata.set_quoted('SYSTEM', host_machine.cpu_family() + '-' + host_machine.system()) deps_private_maybe_subproject = [ ] @@ -73,6 +73,7 @@ subdir('build-utils-meson/threads') boost = dependency( 'boost', modules : ['container'], + include_type: 'system', ) # boost is a public dependency, but not a pkg-config dependency unfortunately, so we # put in `deps_other`. @@ -113,7 +114,7 @@ if aws_s3.found() '-laws-cpp-sdk-core', '-laws-crt-cpp', ], - ) + ).as_system('system') endif deps_other += aws_s3 @@ -327,7 +328,6 @@ prefix = get_option('prefix') path_opts = [ # Meson built-ins. 'datadir', - 'bindir', 'mandir', 'libdir', 'includedir', @@ -372,7 +372,6 @@ cpp_str_defines = { 'NIX_STATE_DIR': state_dir / 'nix', 'NIX_LOG_DIR': log_dir, 'NIX_CONF_DIR': sysconfdir / 'nix', - 'NIX_BIN_DIR': bindir, 'NIX_MAN_DIR': mandir, } diff --git a/src/libstore/names.cc b/src/libstore/names.cc index 277aabf0f..c0e1b1022 100644 --- a/src/libstore/names.cc +++ b/src/libstore/names.cc @@ -94,7 +94,7 @@ static bool componentsLT(const std::string_view c1, const std::string_view c2) } -int compareVersions(const std::string_view v1, const std::string_view v2) +std::strong_ordering compareVersions(const std::string_view v1, const std::string_view v2) { auto p1 = v1.begin(); auto p2 = v2.begin(); @@ -102,11 +102,11 @@ int compareVersions(const std::string_view v1, const std::string_view v2) while (p1 != v1.end() || p2 != v2.end()) { auto c1 = nextComponent(p1, v1.end()); auto c2 = nextComponent(p2, v2.end()); - if (componentsLT(c1, c2)) return -1; - else if (componentsLT(c2, c1)) return 1; + if (componentsLT(c1, c2)) return std::strong_ordering::less; + else if (componentsLT(c2, c1)) return std::strong_ordering::greater; } - return 0; + return std::strong_ordering::equal; } diff --git a/src/libstore/names.hh b/src/libstore/names.hh index d82b99bb4..a6909d545 100644 --- a/src/libstore/names.hh +++ b/src/libstore/names.hh @@ -30,7 +30,7 @@ typedef std::list DrvNames; std::string_view nextComponent(std::string_view::const_iterator & p, const std::string_view::const_iterator end); -int compareVersions(const std::string_view v1, const std::string_view v2); +std::strong_ordering compareVersions(const std::string_view v1, const std::string_view v2); DrvNames drvNamesFromArgs(const Strings & opArgs); } diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 288f618d5..83e63794e 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -164,7 +164,7 @@ public: Cache & getCache(State & state, const std::string & uri) { auto i = state.caches.find(uri); - if (i == state.caches.end()) abort(); + if (i == state.caches.end()) unreachable(); return i->second; } @@ -211,7 +211,7 @@ public: { auto r(state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority)); - if (!r.next()) { abort(); } + if (!r.next()) { unreachable(); } ret.id = (int) r.getInt(0); } diff --git a/src/libstore/package.nix b/src/libstore/package.nix index 0a2ace91e..4582ba0d2 100644 --- a/src/libstore/package.nix +++ b/src/libstore/package.nix @@ -66,10 +66,7 @@ mkMesonDerivation (finalAttrs: { ] ++ lib.optional stdenv.hostPlatform.isLinux libseccomp # There have been issues building these dependencies ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin)) - (aws-sdk-cpp.override { - apis = ["s3" "transfer"]; - customMemoryManagement = false; - }) + aws-sdk-cpp ; propagatedBuildInputs = [ @@ -101,12 +98,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index ebb0864c5..555936c18 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -73,8 +73,11 @@ void RemoteStore::initConnection(Connection & conn) StringSink saved; TeeSource tee(conn.from, saved); try { - conn.protoVersion = WorkerProto::BasicClientConnection::handshake( - conn.to, tee, PROTOCOL_VERSION); + auto [protoVersion, features] = WorkerProto::BasicClientConnection::handshake( + conn.to, tee, PROTOCOL_VERSION, + WorkerProto::allFeatures); + conn.protoVersion = protoVersion; + conn.features = features; } catch (SerialisationError & e) { /* In case the other side is waiting for our input, close it. */ @@ -88,6 +91,9 @@ void RemoteStore::initConnection(Connection & conn) static_cast(conn) = conn.postHandshake(*this); + for (auto & feature : conn.features) + debug("negotiated feature '%s'", feature); + auto ex = conn.processStderrReturn(); if (ex) std::rethrow_exception(ex); } diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 92ab47cd6..21175b1eb 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -220,8 +220,6 @@ std::string S3BinaryCacheStoreConfig::doc() struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStore { - std::string bucketName; - Stats stats; S3Helper s3Helper; diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh index f0aeb3bed..18de115ae 100644 --- a/src/libstore/s3.hh +++ b/src/libstore/s3.hh @@ -8,7 +8,7 @@ #include #include -namespace Aws { namespace Client { class ClientConfiguration; } } +namespace Aws { namespace Client { struct ClientConfiguration; } } namespace Aws { namespace S3 { class S3Client; } } namespace nix { diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 2c4dee518..b3e5ad014 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -920,7 +920,7 @@ StorePathSet Store::exportReferences(const StorePathSet & storePaths, const Stor const Store::Stats & Store::getStats() { { - auto state_(state.lock()); + auto state_(state.readLock()); stats.pathInfoCacheSize = state_->pathInfoCache.size(); } return stats; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 7d5f533c5..8288cfdf0 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -201,7 +201,7 @@ protected: LRUCache pathInfoCache; }; - Sync state; + SharedSync state; std::shared_ptr diskCache; diff --git a/src/libstore/unix/build/hook-instance.cc b/src/libstore/unix/build/hook-instance.cc index d73d86ff2..4f8492fe9 100644 --- a/src/libstore/unix/build/hook-instance.cc +++ b/src/libstore/unix/build/hook-instance.cc @@ -4,6 +4,7 @@ #include "file-system.hh" #include "child.hh" #include "strings.hh" +#include "executable-path.hh" namespace nix { @@ -16,11 +17,18 @@ HookInstance::HookInstance() if (buildHookArgs.empty()) throw Error("'build-hook' setting is empty"); - auto buildHook = canonPath(buildHookArgs.front()); + std::filesystem::path buildHook = buildHookArgs.front(); buildHookArgs.pop_front(); + try { + buildHook = ExecutablePath::load().findPath(buildHook); + } catch (ExecutableLookupError & e) { + e.addTrace(nullptr, "while resolving the 'build-hook' setting'"); + throw; + } + Strings args; - args.push_back(std::string(baseNameOf(buildHook))); + args.push_back(buildHook.filename().string()); for (auto & arg : buildHookArgs) args.push_back(arg); @@ -59,7 +67,7 @@ HookInstance::HookInstance() if (dup2(builderOut.readSide.get(), 5) == -1) throw SysError("dupping builder's stdout/stderr"); - execv(buildHook.c_str(), stringsToCharPtrs(args).data()); + execv(buildHook.native().c_str(), stringsToCharPtrs(args).data()); throw SysError("executing '%s'", buildHook); }); diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 0dd102200..d3482df17 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -165,7 +165,7 @@ void LocalDerivationGoal::killSandbox(bool getStats) buildResult.cpuSystem = stats.cpuSystem; } #else - abort(); + unreachable(); #endif } @@ -1258,7 +1258,7 @@ bool LocalDerivationGoal::isAllowed(const DerivedPath & req) struct RestrictedStoreConfig : virtual LocalFSStoreConfig { using LocalFSStoreConfig::LocalFSStoreConfig; - const std::string name() { return "Restricted Store"; } + const std::string name() override { return "Restricted Store"; } }; /* A wrapper around LocalStore that only allows building/querying of @@ -1702,10 +1702,13 @@ void setupSeccomp() throw SysError("unable to add seccomp rule"); } - /* Prevent builders from creating EAs or ACLs. Not all filesystems + /* Prevent builders from using EAs or ACLs. Not all filesystems support these, and they're not allowed in the Nix store because they're not representable in the NAR serialisation. */ - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 || + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(getxattr), 0) != 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lgetxattr), 0) != 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fgetxattr), 0) != 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 || seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 || seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0) throw SysError("unable to add seccomp rule"); diff --git a/src/libstore/unix/pathlocks.cc b/src/libstore/unix/pathlocks.cc index af21319a7..1ec4579ec 100644 --- a/src/libstore/unix/pathlocks.cc +++ b/src/libstore/unix/pathlocks.cc @@ -45,7 +45,7 @@ bool lockFile(Descriptor desc, LockType lockType, bool wait) if (lockType == ltRead) type = LOCK_SH; else if (lockType == ltWrite) type = LOCK_EX; else if (lockType == ltNone) type = LOCK_UN; - else abort(); + else unreachable(); if (wait) { while (flock(desc, type) != 0) { diff --git a/src/libstore/worker-protocol-connection.cc b/src/libstore/worker-protocol-connection.cc index 93d13d48e..a47dbb689 100644 --- a/src/libstore/worker-protocol-connection.cc +++ b/src/libstore/worker-protocol-connection.cc @@ -5,6 +5,8 @@ namespace nix { +const std::set WorkerProto::allFeatures{}; + WorkerProto::BasicClientConnection::~BasicClientConnection() { try { @@ -137,8 +139,21 @@ void WorkerProto::BasicClientConnection::processStderr(bool * daemonException, S } } -WorkerProto::Version -WorkerProto::BasicClientConnection::handshake(BufferedSink & to, Source & from, WorkerProto::Version localVersion) +static std::set +intersectFeatures(const std::set & a, const std::set & b) +{ + std::set res; + for (auto & x : a) + if (b.contains(x)) + res.insert(x); + return res; +} + +std::tuple> WorkerProto::BasicClientConnection::handshake( + BufferedSink & to, + Source & from, + WorkerProto::Version localVersion, + const std::set & supportedFeatures) { to << WORKER_MAGIC_1 << localVersion; to.flush(); @@ -153,11 +168,24 @@ WorkerProto::BasicClientConnection::handshake(BufferedSink & to, Source & from, if (GET_PROTOCOL_MINOR(daemonVersion) < 10) throw Error("the Nix daemon version is too old"); - return std::min(daemonVersion, localVersion); + auto protoVersion = std::min(daemonVersion, localVersion); + + /* Exchange features. */ + std::set daemonFeatures; + if (GET_PROTOCOL_MINOR(protoVersion) >= 38) { + to << supportedFeatures; + to.flush(); + daemonFeatures = readStrings>(from); + } + + return {protoVersion, intersectFeatures(daemonFeatures, supportedFeatures)}; } -WorkerProto::Version -WorkerProto::BasicServerConnection::handshake(BufferedSink & to, Source & from, WorkerProto::Version localVersion) +std::tuple> WorkerProto::BasicServerConnection::handshake( + BufferedSink & to, + Source & from, + WorkerProto::Version localVersion, + const std::set & supportedFeatures) { unsigned int magic = readInt(from); if (magic != WORKER_MAGIC_1) @@ -165,7 +193,18 @@ WorkerProto::BasicServerConnection::handshake(BufferedSink & to, Source & from, to << WORKER_MAGIC_2 << localVersion; to.flush(); auto clientVersion = readInt(from); - return std::min(clientVersion, localVersion); + + auto protoVersion = std::min(clientVersion, localVersion); + + /* Exchange features. */ + std::set clientFeatures; + if (GET_PROTOCOL_MINOR(protoVersion) >= 38) { + clientFeatures = readStrings>(from); + to << supportedFeatures; + to.flush(); + } + + return {protoVersion, intersectFeatures(clientFeatures, supportedFeatures)}; } WorkerProto::ClientHandshakeInfo WorkerProto::BasicClientConnection::postHandshake(const StoreDirConfig & store) diff --git a/src/libstore/worker-protocol-connection.hh b/src/libstore/worker-protocol-connection.hh index 38287d08e..9c96195b5 100644 --- a/src/libstore/worker-protocol-connection.hh +++ b/src/libstore/worker-protocol-connection.hh @@ -23,6 +23,11 @@ struct WorkerProto::BasicConnection */ WorkerProto::Version protoVersion; + /** + * The set of features that both sides support. + */ + std::set features; + /** * Coercion to `WorkerProto::ReadConn`. This makes it easy to use the * factored out serve protocol serializers with a @@ -72,8 +77,8 @@ struct WorkerProto::BasicClientConnection : WorkerProto::BasicConnection /** * Establishes connection, negotiating version. * - * @return the version provided by the other side of the - * connection. + * @return the minimum version supported by both sides and the set + * of protocol features supported by both sides. * * @param to Taken by reference to allow for various error handling * mechanisms. @@ -82,8 +87,15 @@ struct WorkerProto::BasicClientConnection : WorkerProto::BasicConnection * handling mechanisms. * * @param localVersion Our version which is sent over + * + * @param features The protocol features that we support */ - static Version handshake(BufferedSink & to, Source & from, WorkerProto::Version localVersion); + // FIXME: this should probably be a constructor. + static std::tuple> handshake( + BufferedSink & to, + Source & from, + WorkerProto::Version localVersion, + const std::set & supportedFeatures); /** * After calling handshake, must call this to exchange some basic @@ -138,8 +150,15 @@ struct WorkerProto::BasicServerConnection : WorkerProto::BasicConnection * handling mechanisms. * * @param localVersion Our version which is sent over + * + * @param features The protocol features that we support */ - static WorkerProto::Version handshake(BufferedSink & to, Source & from, WorkerProto::Version localVersion); + // FIXME: this should probably be a constructor. + static std::tuple> handshake( + BufferedSink & to, + Source & from, + WorkerProto::Version localVersion, + const std::set & supportedFeatures); /** * After calling handshake, must call this to exchange some basic diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 9fc16d015..c356fa1bf 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -11,7 +11,9 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION (1 << 8 | 37) +/* Note: you generally shouldn't change the protocol version. Define a + new `WorkerProto::Feature` instead. */ +#define PROTOCOL_VERSION (1 << 8 | 38) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -131,6 +133,10 @@ struct WorkerProto { WorkerProto::Serialise::write(store, conn, t); } + + using Feature = std::string; + + static const std::set allFeatures; }; enum struct WorkerProto::Op : uint64_t diff --git a/src/libutil-c/nix_api_util.h b/src/libutil-c/nix_api_util.h index e0ca04e69..ad6f32859 100644 --- a/src/libutil-c/nix_api_util.h +++ b/src/libutil-c/nix_api_util.h @@ -56,47 +56,51 @@ extern "C" { * - NIX_ERR_KEY: A key error occurred (-3) * - NIX_ERR_NIX_ERROR: A generic Nix error occurred (-4) */ -typedef int nix_err; +enum nix_err { -/** - * @brief No error occurred. - * - * This error code is returned when no error has occurred during the function - * execution. - */ -#define NIX_OK 0 + /** + * @brief No error occurred. + * + * This error code is returned when no error has occurred during the function + * execution. + */ + NIX_OK = 0, -/** - * @brief An unknown error occurred. - * - * This error code is returned when an unknown error occurred during the - * function execution. - */ -#define NIX_ERR_UNKNOWN -1 + /** + * @brief An unknown error occurred. + * + * This error code is returned when an unknown error occurred during the + * function execution. + */ + NIX_ERR_UNKNOWN = -1, -/** - * @brief An overflow error occurred. - * - * This error code is returned when an overflow error occurred during the - * function execution. - */ -#define NIX_ERR_OVERFLOW -2 + /** + * @brief An overflow error occurred. + * + * This error code is returned when an overflow error occurred during the + * function execution. + */ + NIX_ERR_OVERFLOW = -2, -/** - * @brief A key error occurred. - * - * This error code is returned when a key error occurred during the function - * execution. - */ -#define NIX_ERR_KEY -3 + /** + * @brief A key error occurred. + * + * This error code is returned when a key error occurred during the function + * execution. + */ + NIX_ERR_KEY = -3, -/** - * @brief A generic Nix error occurred. - * - * This error code is returned when a generic Nix error occurred during the - * function execution. - */ -#define NIX_ERR_NIX_ERROR -4 + /** + * @brief A generic Nix error occurred. + * + * This error code is returned when a generic Nix error occurred during the + * function execution. + */ + NIX_ERR_NIX_ERROR = -4, + +}; + +typedef enum nix_err nix_err; /** * @brief This object stores error state. diff --git a/src/libutil-c/package.nix b/src/libutil-c/package.nix index 53451998d..ccfafd4d3 100644 --- a/src/libutil-c/package.nix +++ b/src/libutil-c/package.nix @@ -62,12 +62,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libutil/checked-arithmetic.hh b/src/libutil/checked-arithmetic.hh new file mode 100644 index 000000000..55d6ad205 --- /dev/null +++ b/src/libutil/checked-arithmetic.hh @@ -0,0 +1,182 @@ +#pragma once +/** + * @file Checked arithmetic with classes that make it hard to accidentally make something an unchecked operation. + */ + +#include +#include // IWYU pragma: keep +#include +#include +#include +#include +#include + +namespace nix::checked { + +class DivideByZero : std::exception +{}; + +/** + * Numeric value enforcing checked arithmetic. Performing mathematical operations on such values will return a Result + * type which needs to be checked. + */ +template +struct Checked +{ + using Inner = T; + + // TODO: this must be a "trivial default constructor", which means it + // cannot set the value to NOT DO UB on uninit. + T value; + + Checked() = default; + explicit Checked(T const value) + : value{value} + { + } + Checked(Checked const & other) = default; + Checked(Checked && other) = default; + Checked & operator=(Checked const & other) = default; + + std::strong_ordering operator<=>(Checked const & other) const = default; + std::strong_ordering operator<=>(T const & other) const + { + return value <=> other; + } + + explicit operator T() const + { + return value; + } + + enum class OverflowKind { + NoOverflow, + Overflow, + DivByZero, + }; + + class Result + { + T value; + OverflowKind overflowed_; + + public: + Result(T value, bool overflowed) + : value{value} + , overflowed_{overflowed ? OverflowKind::Overflow : OverflowKind::NoOverflow} + { + } + Result(T value, OverflowKind overflowed) + : value{value} + , overflowed_{overflowed} + { + } + + bool operator==(Result other) const + { + return value == other.value && overflowed_ == other.overflowed_; + } + + std::optional valueChecked() const + { + if (overflowed_ != OverflowKind::NoOverflow) { + return std::nullopt; + } else { + return value; + } + } + + /** + * Returns the result as if the arithmetic were performed as wrapping arithmetic. + * + * \throws DivideByZero if the operation was a divide by zero. + */ + T valueWrapping() const + { + if (overflowed_ == OverflowKind::DivByZero) { + throw DivideByZero{}; + } + return value; + } + + bool overflowed() const + { + return overflowed_ == OverflowKind::Overflow; + } + + bool divideByZero() const + { + return overflowed_ == OverflowKind::DivByZero; + } + }; + + Result operator+(Checked const other) const + { + return (*this) + other.value; + } + Result operator+(T const other) const + { + T result; + bool overflowed = __builtin_add_overflow(value, other, &result); + return Result{result, overflowed}; + } + + Result operator-(Checked const other) const + { + return (*this) - other.value; + } + Result operator-(T const other) const + { + T result; + bool overflowed = __builtin_sub_overflow(value, other, &result); + return Result{result, overflowed}; + } + + Result operator*(Checked const other) const + { + return (*this) * other.value; + } + Result operator*(T const other) const + { + T result; + bool overflowed = __builtin_mul_overflow(value, other, &result); + return Result{result, overflowed}; + } + + Result operator/(Checked const other) const + { + return (*this) / other.value; + } + /** + * Performs a checked division. + * + * If the right hand side is zero, the result is marked as a DivByZero and + * valueWrapping will throw. + */ + Result operator/(T const other) const + { + constexpr T const minV = std::numeric_limits::min(); + + // It's only possible to overflow with signed division since doing so + // requires crossing the two's complement limits by MIN / -1 (since + // two's complement has one more in range in the negative direction + // than in the positive one). + if (std::is_signed() && (value == minV && other == -1)) { + return Result{minV, true}; + } else if (other == 0) { + return Result{0, OverflowKind::DivByZero}; + } else { + T result = value / other; + return Result{result, false}; + } + } +}; + +template +std::ostream & operator<<(std::ostream & ios, Checked v) +{ + ios << v.value; + return ios; +} + +} diff --git a/src/libutil/chunked-vector.hh b/src/libutil/chunked-vector.hh index d914e2542..4709679a6 100644 --- a/src/libutil/chunked-vector.hh +++ b/src/libutil/chunked-vector.hh @@ -6,6 +6,8 @@ #include #include +#include "error.hh" + namespace nix { /** @@ -30,7 +32,7 @@ private: auto & addChunk() { if (size_ >= std::numeric_limits::max() - ChunkSize) - abort(); + unreachable(); chunks.emplace_back(); chunks.back().reserve(ChunkSize); return chunks.back(); diff --git a/src/libutil/config.cc b/src/libutil/config.cc index aa0e73673..a6918692b 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -1,6 +1,7 @@ #include "config.hh" #include "args.hh" #include "abstract-setting-to-json.hh" +#include "environment-variables.hh" #include "experimental-features.hh" #include "util.hh" #include "file-system.hh" @@ -170,9 +171,18 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string set(name, value); // Then apply other settings - for (const auto & [name, value] : parsedContents) - if (name != "experimental-features" && name != "extra-experimental-features") + // XXX: NIX_PATH must override the regular setting! This is done in `initGC()` + // Environment variables overriding settings should probably be part of the Config mechanism, + // but at the time of writing it's not worth building that for just one thing + for (const auto & [name, value] : parsedContents) { + if (name != "experimental-features" && name != "extra-experimental-features") { + if ((name == "nix-path" || name == "extra-nix-path") + && getEnv("NIX_PATH").has_value()) { + continue; + } set(name, value); + } + } } void Config::resetOverridden() diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 1952ba1b8..c0c59ac68 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -393,7 +393,7 @@ struct ExperimentalFeatureSettings : Config { {{#include experimental-features-shortlist.md}} - Experimental features are [further documented in the manual](@docroot@/contributing/experimental-features.md). + Experimental features are [further documented in the manual](@docroot@/development/experimental-features.md). )"}; /** diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc index 6ca48220d..0bc46d746 100644 --- a/src/libutil/current-process.cc +++ b/src/libutil/current-process.cc @@ -15,13 +15,12 @@ #if __linux__ # include -# include # include "cgroup.hh" # include "namespaces.hh" #endif #ifndef _WIN32 -# include +# include #endif namespace nix { @@ -138,7 +137,7 @@ std::optional getSelfExe() { static auto cached = []() -> std::optional { - #if __linux__ + #if __linux__ || __GNU__ return readLink("/proc/self/exe"); #elif __APPLE__ char buf[1024]; diff --git a/src/libutil/environment-variables.cc b/src/libutil/environment-variables.cc index d43197aa0..5947cf742 100644 --- a/src/libutil/environment-variables.cc +++ b/src/libutil/environment-variables.cc @@ -1,20 +1,23 @@ #include "util.hh" #include "environment-variables.hh" -extern char * * environ __attribute__((weak)); +extern char ** environ __attribute__((weak)); namespace nix { std::optional getEnv(const std::string & key) { char * value = getenv(key.c_str()); - if (!value) return {}; + if (!value) + return {}; return std::string(value); } -std::optional getEnvNonEmpty(const std::string & key) { +std::optional getEnvNonEmpty(const std::string & key) +{ auto value = getEnv(key); - if (value == "") return {}; + if (value == "") + return {}; return value; } diff --git a/src/libutil/environment-variables.hh b/src/libutil/environment-variables.hh index e0649adac..879e1f304 100644 --- a/src/libutil/environment-variables.hh +++ b/src/libutil/environment-variables.hh @@ -9,6 +9,7 @@ #include #include "types.hh" +#include "file-path.hh" namespace nix { @@ -17,6 +18,11 @@ namespace nix { */ std::optional getEnv(const std::string & key); +/** + * Like `getEnv`, but using `OsString` to avoid coercions. + */ +std::optional getEnvOs(const OsString & key); + /** * @return a non empty environment variable. Returns nullopt if the env * variable is set to "" @@ -43,6 +49,11 @@ int unsetenv(const char * name); */ int setEnv(const char * name, const char * value); +/** + * Like `setEnv`, but using `OsString` to avoid coercions. + */ +int setEnvOs(const OsString & name, const OsString & value); + /** * Clear the environment. */ diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 33c391963..ccd008c7c 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -1,3 +1,5 @@ +#include + #include "error.hh" #include "environment-variables.hh" #include "signals.hh" @@ -430,4 +432,36 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s return out; } +/** Write to stderr in a robust and minimal way, considering that the process + * may be in a bad state. + */ +static void writeErr(std::string_view buf) +{ + while (!buf.empty()) { + auto n = write(STDERR_FILENO, buf.data(), buf.size()); + if (n < 0) { + if (errno == EINTR) continue; + abort(); + } + buf = buf.substr(n); + } +} + +void panic(std::string_view msg) +{ + writeErr("\n\n" ANSI_RED "terminating due to unexpected unrecoverable internal error: " ANSI_NORMAL ); + writeErr(msg); + writeErr("\n"); + abort(); +} + +void panic(const char * file, int line, const char * func) +{ + char buf[512]; + int n = snprintf(buf, sizeof(buf), "Unexpected condition in %s at %s:%d", func, file, line); + if (n < 0) + panic("Unexpected condition and could not format error message"); + panic(std::string_view(buf, std::min(static_cast(sizeof(buf)), n))); +} + } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index d7fe902d6..58d902622 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -273,4 +273,24 @@ using NativeSysError = */ void throwExceptionSelfCheck(); +/** + * Print a message and abort(). + */ +[[noreturn]] +void panic(std::string_view msg); + +/** + * Print a basic error message with source position and abort(). + * Use the unreachable() macro to call this. + */ +[[noreturn]] +void panic(const char * file, int line, const char * func); + +/** + * Print a basic error message with source position and abort(). + * + * @note: This assumes that the logger is operational + */ +#define unreachable() (::nix::panic(__FILE__, __LINE__, __func__)) + } diff --git a/src/libutil/executable-path.cc b/src/libutil/executable-path.cc new file mode 100644 index 000000000..8005a19be --- /dev/null +++ b/src/libutil/executable-path.cc @@ -0,0 +1,95 @@ +#include "environment-variables.hh" +#include "executable-path.hh" +#include "strings-inline.hh" +#include "util.hh" +#include "file-path-impl.hh" + +namespace nix { + +namespace fs = std::filesystem; + +constexpr static const OsStringView path_var_separator{ + &ExecutablePath::separator, + 1, +}; + +ExecutablePath ExecutablePath::load() +{ + // "If PATH is unset or is set to null, the path search is + // implementation-defined." + // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03 + return ExecutablePath::parse(getEnvOs(OS_STR("PATH")).value_or(OS_STR(""))); +} + +ExecutablePath ExecutablePath::parse(const OsString & path) +{ + auto strings = path.empty() ? (std::list{}) + : basicSplitString, OsString::value_type>(path, path_var_separator); + + std::vector ret; + ret.reserve(strings.size()); + + std::transform( + std::make_move_iterator(strings.begin()), + std::make_move_iterator(strings.end()), + std::back_inserter(ret), + [](auto && str) { + return fs::path{ + str.empty() + // "A zero-length prefix is a legacy feature that + // indicates the current working directory. It + // appears as two adjacent characters + // ("::"), as an initial preceding the rest + // of the list, or as a trailing following + // the rest of the list." + // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03 + ? OS_STR(".") + : std::move(str), + }; + }); + + return {ret}; +} + +OsString ExecutablePath::render() const +{ + std::vector path2; + for (auto & p : directories) + path2.push_back(p.native()); + return basicConcatStringsSep(path_var_separator, path2); +} + +std::optional +ExecutablePath::findName(const OsString & exe, std::function isExecutable) const +{ + // "If the pathname being sought contains a , the search + // through the path prefixes shall not be performed." + // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03 + assert(OsPathTrait::rfindPathSep(exe) == exe.npos); + + for (auto & dir : directories) { + auto candidate = dir / exe; + if (isExecutable(candidate)) + return std::filesystem::canonical(candidate); + } + + return std::nullopt; +} + +fs::path ExecutablePath::findPath(const fs::path & exe, std::function isExecutable) const +{ + // "If the pathname being sought contains a , the search + // through the path prefixes shall not be performed." + // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03 + if (exe.filename() == exe) { + auto resOpt = findName(exe, isExecutable); + if (resOpt) + return *resOpt; + else + throw ExecutableLookupError("Could not find executable '%s'", exe.native()); + } else { + return exe; + } +} + +} // namespace nix diff --git a/src/libutil/executable-path.hh b/src/libutil/executable-path.hh new file mode 100644 index 000000000..f46d5e212 --- /dev/null +++ b/src/libutil/executable-path.hh @@ -0,0 +1,77 @@ +#pragma once +///@file + +#include "file-system.hh" + +namespace nix { + +MakeError(ExecutableLookupError, Error); + +struct ExecutablePath +{ + std::vector directories; + + constexpr static const OsString::value_type separator = +#ifdef WIN32 + L';' +#else + ':' +#endif + ; + + /** + * Parse `path` into a list of paths. + * + * On Unix we split on `:`, on Windows we split on `;`. + * + * For Unix, this is according to the POSIX spec for `PATH`. + * https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03 + */ + static ExecutablePath parse(const OsString & path); + + /** + * Load the `PATH` environment variable and `parse` it. + */ + static ExecutablePath load(); + + /** + * Opposite of `parse` + */ + OsString render() const; + + /** + * Search for an executable. + * + * For Unix, this is according to the POSIX spec for `PATH`. + * https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03 + * + * @param exe This must just be a name, and not contain any `/` (or + * `\` on Windows). in case it does, per the spec no lookup should + * be perfomed, and the path (it is not just a file name) as is. + * This is the caller's respsonsibility. + * + * This is a pure function, except for the default `isExecutable` + * argument, which uses the ambient file system to check if a file is + * executable (and exists). + * + * @return path to a resolved executable + */ + std::optional findName( + const OsString & exe, + std::function isExecutableFile = isExecutableFileAmbient) const; + + /** + * Like the `findName` but also allows a file path as input. + * + * This implements the full POSIX spec: if the path is just a name, + * it searches like the above. Otherwise, it returns the path as is. + * If (in the name case) the search fails, an exception is thrown. + */ + std::filesystem::path findPath( + const std::filesystem::path & exe, + std::function isExecutable = isExecutableFileAmbient) const; + + bool operator==(const ExecutablePath &) const = default; +}; + +} // namespace nix diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 1c080e372..a0c955816 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -24,7 +24,7 @@ struct ExperimentalFeatureDetails * feature, we either have no issue at all if few features are not added * at the end of the list, or a proper merge conflict if they are. */ -constexpr size_t numXpFeatures = 1 + static_cast(Xp::VerifiedFetches); +constexpr size_t numXpFeatures = 1 + static_cast(Xp::PipeOperators); constexpr std::array xpFeatureDetails = {{ { @@ -294,6 +294,14 @@ constexpr std::array xpFeatureDetails )", .trackingUrl = "https://github.com/NixOS/nix/milestone/48", }, + { + .tag = Xp::PipeOperators, + .name = "pipe-operators", + .description = R"( + Add `|>` and `<|` operators to the Nix language. + )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/55", + }, }}; static_assert( diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 6ffbc0c10..e65e51280 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -35,6 +35,7 @@ enum struct ExperimentalFeature ConfigurableImpureEnv, MountedSSHStore, VerifiedFetches, + PipeOperators, }; /** diff --git a/src/libutil/file-content-address.cc b/src/libutil/file-content-address.cc index 438dac7da..86378dd67 100644 --- a/src/libutil/file-content-address.cc +++ b/src/libutil/file-content-address.cc @@ -63,7 +63,7 @@ std::string_view renderFileIngestionMethod(FileIngestionMethod method) case FileIngestionMethod::Git: return "git"; default: - abort(); + unreachable(); } } diff --git a/src/libutil/file-path-impl.hh b/src/libutil/file-path-impl.hh index 4c90150fd..d7c823fd0 100644 --- a/src/libutil/file-path-impl.hh +++ b/src/libutil/file-path-impl.hh @@ -91,13 +91,10 @@ struct WindowsPathTrait }; -/** - * @todo Revisit choice of `char` or `wchar_t` for `WindowsPathTrait` - * argument. - */ -using NativePathTrait = +template +using OsPathTrait = #ifdef _WIN32 - WindowsPathTrait + WindowsPathTrait #else UnixPathTrait #endif diff --git a/src/libutil/file-path.hh b/src/libutil/file-path.hh index 6589c4060..8e4a88b9d 100644 --- a/src/libutil/file-path.hh +++ b/src/libutil/file-path.hh @@ -1,10 +1,10 @@ #pragma once ///@file -#include #include #include "types.hh" +#include "os-string.hh" namespace nix { @@ -22,39 +22,26 @@ typedef std::set PathSetNG; * * @todo drop `NG` suffix and replace the one in `types.hh`. */ -struct PathViewNG : std::basic_string_view +struct PathViewNG : OsStringView { - using string_view = std::basic_string_view; + using string_view = OsStringView; using string_view::string_view; PathViewNG(const std::filesystem::path & path) - : std::basic_string_view(path.native()) + : OsStringView{path.native()} { } - PathViewNG(const std::filesystem::path::string_type & path) - : std::basic_string_view(path) + PathViewNG(const OsString & path) + : OsStringView{path} { } const string_view & native() const { return *this; } string_view & native() { return *this; } }; -std::string os_string_to_string(PathViewNG::string_view path); - -std::filesystem::path::string_type string_to_os_string(std::string_view s); - std::optional maybePath(PathView path); std::filesystem::path pathNG(PathView path); -/** - * Create string literals with the native character width of paths - */ -#ifndef _WIN32 -# define PATHNG_LITERAL(s) s -#else -# define PATHNG_LITERAL(s) L ## s -#endif - } diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 060a806fb..536ae29ab 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -92,7 +92,7 @@ Path canonPath(PathView path, bool resolveSymlinks) arbitrary (but high) limit to prevent infinite loops. */ unsigned int followCount = 0, maxFollow = 1024; - auto ret = canonPathInner( + auto ret = canonPathInner>( path, [&followCount, &temp, maxFollow, resolveSymlinks] (std::string & result, std::string_view & remaining) { @@ -122,7 +122,7 @@ Path canonPath(PathView path, bool resolveSymlinks) Path dirOf(const PathView path) { - Path::size_type pos = NativePathTrait::rfindPathSep(path); + Path::size_type pos = OsPathTrait::rfindPathSep(path); if (pos == path.npos) return "."; return fs::path{path}.parent_path().string(); @@ -135,10 +135,10 @@ std::string_view baseNameOf(std::string_view path) return ""; auto last = path.size() - 1; - while (last > 0 && NativePathTrait::isPathSep(path[last])) + while (last > 0 && OsPathTrait::isPathSep(path[last])) last -= 1; - auto pos = NativePathTrait::rfindPathSep(path, last); + auto pos = OsPathTrait::rfindPathSep(path, last); if (pos == path.npos) pos = 0; else @@ -569,7 +569,7 @@ void replaceSymlink(const Path & target, const Path & link) } void setWriteTime( - const std::filesystem::path & path, + const fs::path & path, time_t accessedTime, time_t modificationTime, std::optional optIsSymlink) @@ -685,4 +685,18 @@ void moveFile(const Path & oldName, const Path & newName) ////////////////////////////////////////////////////////////////////// +bool isExecutableFileAmbient(const fs::path & exe) { + // Check file type, because directory being executable means + // something completely different. + // `is_regular_file` follows symlinks before checking. + return std::filesystem::is_regular_file(exe) + && access(exe.string().c_str(), +#ifdef WIN32 + 0 // TODO do better +#else + X_OK +#endif + ) == 0; +} + } diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh index 4b215162d..1ae5fa136 100644 --- a/src/libutil/file-system.hh +++ b/src/libutil/file-system.hh @@ -263,6 +263,12 @@ std::pair createTempFile(const Path & prefix = "nix"); */ Path defaultTempDir(); +/** + * Interpret `exe` as a location in the ambient file system and return + * whether it resolves to a file that is executable. + */ +bool isExecutableFileAmbient(const std::filesystem::path & exe); + /** * Used in various places. */ diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc index 3246e0902..f15324d0a 100644 --- a/src/libutil/fs-sink.cc +++ b/src/libutil/fs-sink.cc @@ -53,7 +53,7 @@ void copyRecursive( throw Error("file '%1%' has an unsupported type", from); default: - abort(); + unreachable(); } } diff --git a/src/libutil/git.cc b/src/libutil/git.cc index a6968a43e..af91fa643 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -201,7 +201,7 @@ std::optional convertMode(SourceAccessor::Type type) case SourceAccessor::tRegular: return Mode::Regular; case SourceAccessor::tDirectory: return Mode::Directory; case SourceAccessor::tMisc: return std::nullopt; - default: abort(); + default: unreachable(); } } diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index 35b913e42..ab2a8695d 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -25,7 +25,7 @@ static size_t regularHashSize(HashAlgorithm type) { case HashAlgorithm::SHA256: return sha256HashSize; case HashAlgorithm::SHA512: return sha512HashSize; } - abort(); + unreachable(); } diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 55751b4cf..29427f2f6 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -189,7 +189,7 @@ struct JSONLogger : Logger { else if (f.type == Logger::Field::tString) arr.push_back(f.s); else - abort(); + unreachable(); } void write(const nlohmann::json & json) diff --git a/src/libutil/lru-cache.hh b/src/libutil/lru-cache.hh index 0e19517ed..6e14cac35 100644 --- a/src/libutil/lru-cache.hh +++ b/src/libutil/lru-cache.hh @@ -89,7 +89,7 @@ public: return i->second.second; } - size_t size() + size_t size() const { return data.size(); } diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 04c778c31..200eeb4e9 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -62,6 +62,7 @@ endif boost = dependency( 'boost', modules : ['context', 'coroutine'], + include_type: 'system', ) # boost is a public dependency, but not a pkg-config dependency unfortunately, so we # put in `deps_other`. @@ -128,6 +129,7 @@ sources = files( 'english.cc', 'environment-variables.cc', 'error.cc', + 'executable-path.cc', 'exit.cc', 'experimental-features.cc', 'file-content-address.cc', @@ -170,6 +172,7 @@ headers = [config_h] + files( 'args/root.hh', 'callback.hh', 'canon-path.hh', + 'checked-arithmetic.hh', 'chunked-vector.hh', 'closure.hh', 'comparator.hh', @@ -182,6 +185,7 @@ headers = [config_h] + files( 'english.hh', 'environment-variables.hh', 'error.hh', + 'executable-path.hh', 'exit.hh', 'experimental-features.hh', 'file-content-address.hh', @@ -201,6 +205,7 @@ headers = [config_h] + files( 'lru-cache.hh', 'memory-source-accessor.hh', 'muxable-pipe.hh', + 'os-string.hh', 'pool.hh', 'position.hh', 'posix-source-accessor.hh', diff --git a/src/libutil/os-string.hh b/src/libutil/os-string.hh new file mode 100644 index 000000000..0d75173e5 --- /dev/null +++ b/src/libutil/os-string.hh @@ -0,0 +1,43 @@ +#pragma once +///@file + +#include +#include +#include + +namespace nix { + +/** + * Named because it is similar to the Rust type, except it is in the + * native encoding not WTF-8. + * + * Same as `std::filesystem::path::string_type`, but manually defined to + * avoid including a much more complex header. + */ +using OsString = std::basic_string< +#if defined(_WIN32) && !defined(__CYGWIN__) + wchar_t +#else + char +#endif + >; + +/** + * `std::string_view` counterpart for `OsString`. + */ +using OsStringView = std::basic_string_view; + +std::string os_string_to_string(OsStringView path); + +OsString string_to_os_string(std::string_view s); + +/** + * Create string literals with the native character width of paths + */ +#ifndef _WIN32 +# define OS_STR(s) s +#else +# define OS_STR(s) L##s +#endif + +} diff --git a/src/libutil/package.nix b/src/libutil/package.nix index 28d7d8f0e..4ce1a75b0 100644 --- a/src/libutil/package.nix +++ b/src/libutil/package.nix @@ -88,12 +88,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 35047f89e..2b1a485d5 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -97,7 +97,7 @@ std::optional PosixSourceAccessor::cachedLstat(const CanonPath & pa Path absPath = makeAbsPath(path).string(); { - auto cache(_cache.read()); + auto cache(_cache.readLock()); auto i = cache->find(absPath); if (i != cache->end()) return i->second; } diff --git a/src/libutil/ref.hh b/src/libutil/ref.hh index 016fdd74a..3d0e64ab4 100644 --- a/src/libutil/ref.hh +++ b/src/libutil/ref.hh @@ -23,14 +23,14 @@ public: : p(r.p) { } - explicit ref(const std::shared_ptr & p) + explicit ref(const std::shared_ptr & p) : p(p) { if (!p) throw std::invalid_argument("null pointer cast to ref"); } - explicit ref(T * p) + explicit ref(T * p) : p(p) { if (!p) diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 36b99905a..4899134d7 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -190,11 +190,11 @@ struct VirtualStackAllocator { class DefaultStackAllocator : public StackAllocator { boost::coroutines2::default_stack stack; - boost::context::stack_context allocate() { + boost::context::stack_context allocate() override { return stack.allocate(); } - void deallocate(boost::context::stack_context sctx) { + void deallocate(boost::context::stack_context sctx) override { stack.deallocate(sctx); } }; @@ -260,7 +260,7 @@ std::unique_ptr sourceToSink(std::function fun) }); } - if (!*coro) { abort(); } + if (!*coro) { unreachable(); } if (!cur.empty()) { CoroutineContext ctx; @@ -271,12 +271,12 @@ std::unique_ptr sourceToSink(std::function fun) void finish() override { if (!coro) return; - if (!*coro) abort(); + if (!*coro) unreachable(); { CoroutineContext ctx; (*coro)(true); } - if (*coro) abort(); + if (*coro) unreachable(); } }; @@ -316,7 +316,7 @@ std::unique_ptr sinkToSource( }); } - if (!*coro) { eof(); abort(); } + if (!*coro) { eof(); unreachable(); } if (pos == cur.size()) { if (!cur.empty()) { diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 8254a5f89..c7290dcef 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -204,7 +204,7 @@ struct TeeSink : Sink { Sink & sink1, & sink2; TeeSink(Sink & sink1, Sink & sink2) : sink1(sink1), sink2(sink2) { } - virtual void operator () (std::string_view data) + virtual void operator () (std::string_view data) override { sink1(data); sink2(data); @@ -221,7 +221,7 @@ struct TeeSource : Source Sink & sink; TeeSource(Source & orig, Sink & sink) : orig(orig), sink(sink) { } - size_t read(char * data, size_t len) + size_t read(char * data, size_t len) override { size_t n = orig.read(data, len); sink({data, n}); @@ -238,7 +238,7 @@ struct SizedSource : Source size_t remain; SizedSource(Source & orig, size_t size) : orig(orig), remain(size) { } - size_t read(char * data, size_t len) + size_t read(char * data, size_t len) override { if (this->remain <= 0) { throw EndOfFile("sized: unexpected end-of-file"); @@ -483,13 +483,17 @@ struct FramedSource : Source ~FramedSource() { - if (!eof) { - while (true) { - auto n = readInt(from); - if (!n) break; - std::vector data(n); - from(data.data(), n); + try { + if (!eof) { + while (true) { + auto n = readInt(from); + if (!n) break; + std::vector data(n); + from(data.data(), n); + } } + } catch (...) { + ignoreException(); } } diff --git a/src/libutil/strings-inline.hh b/src/libutil/strings-inline.hh index 10c1b19e6..25b8e0ff6 100644 --- a/src/libutil/strings-inline.hh +++ b/src/libutil/strings-inline.hh @@ -4,8 +4,51 @@ namespace nix { +template +C basicTokenizeString(std::basic_string_view s, std::basic_string_view separators) +{ + C result; + auto pos = s.find_first_not_of(separators, 0); + while (pos != s.npos) { + auto end = s.find_first_of(separators, pos + 1); + if (end == s.npos) + end = s.size(); + result.insert(result.end(), std::basic_string(s, pos, end - pos)); + pos = s.find_first_not_of(separators, end); + } + return result; +} + template -std::string concatStringsSep(const std::string_view sep, const C & ss) +C tokenizeString(std::string_view s, std::string_view separators) +{ + return basicTokenizeString(s, separators); +} + +template +C basicSplitString(std::basic_string_view s, std::basic_string_view separators) +{ + C result; + size_t pos = 0; + while (pos <= s.size()) { + auto end = s.find_first_of(separators, pos); + if (end == s.npos) + end = s.size(); + result.insert(result.end(), std::basic_string(s, pos, end - pos)); + pos = end + 1; + } + + return result; +} + +template +C splitString(std::string_view s, std::string_view separators) +{ + return basicSplitString(s, separators); +} + +template +std::basic_string basicConcatStringsSep(const std::basic_string_view sep, const C & ss) { size_t size = 0; bool tail = false; @@ -13,10 +56,10 @@ std::string concatStringsSep(const std::string_view sep, const C & ss) for (const auto & s : ss) { if (tail) size += sep.size(); - size += std::string_view(s).size(); + size += std::basic_string_view{s}.size(); tail = true; } - std::string s; + std::basic_string s; s.reserve(size); tail = false; for (auto & i : ss) { @@ -28,4 +71,36 @@ std::string concatStringsSep(const std::string_view sep, const C & ss) return s; } +template +std::string concatStringsSep(const std::string_view sep, const C & ss) +{ + return basicConcatStringsSep(sep, ss); +} + +template +std::string dropEmptyInitThenConcatStringsSep(const std::string_view sep, const C & ss) +{ + size_t size = 0; + + // TODO? remove to make sure we don't rely on the empty item ignoring behavior, + // or just get rid of this function by understanding the remaining calls. + // for (auto & i : ss) { + // // Make sure we don't rely on the empty item ignoring behavior + // assert(!i.empty()); + // break; + // } + + // need a cast to string_view since this is also called with Symbols + for (const auto & s : ss) + size += sep.size() + std::string_view(s).size(); + std::string s; + s.reserve(size); + for (auto & i : ss) { + if (s.size() != 0) + s += sep; + s += i; + } + return s; +} + } // namespace nix diff --git a/src/libutil/strings.cc b/src/libutil/strings.cc index 7ec618bf4..60297228e 100644 --- a/src/libutil/strings.cc +++ b/src/libutil/strings.cc @@ -1,12 +1,24 @@ +#include #include #include "strings-inline.hh" -#include "util.hh" +#include "os-string.hh" namespace nix { -template std::string concatStringsSep(std::string_view, const Strings &); -template std::string concatStringsSep(std::string_view, const StringSet &); +template std::list tokenizeString(std::string_view s, std::string_view separators); +template std::set tokenizeString(std::string_view s, std::string_view separators); +template std::vector tokenizeString(std::string_view s, std::string_view separators); + +template std::list splitString(std::string_view s, std::string_view separators); +template std::set splitString(std::string_view s, std::string_view separators); +template std::vector splitString(std::string_view s, std::string_view separators); + +template std::list basicSplitString( + std::basic_string_view s, std::basic_string_view separators); + +template std::string concatStringsSep(std::string_view, const std::list &); +template std::string concatStringsSep(std::string_view, const std::set &); template std::string concatStringsSep(std::string_view, const std::vector &); typedef std::string_view strings_2[2]; @@ -16,4 +28,8 @@ template std::string concatStringsSep(std::string_view, const strings_3 &); typedef std::string_view strings_4[4]; template std::string concatStringsSep(std::string_view, const strings_4 &); +template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::list &); +template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::set &); +template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::vector &); + } // namespace nix diff --git a/src/libutil/strings.hh b/src/libutil/strings.hh index 3b112c409..88b48d770 100644 --- a/src/libutil/strings.hh +++ b/src/libutil/strings.hh @@ -8,6 +8,38 @@ namespace nix { +/** + * String tokenizer. + * + * See also `basicSplitString()`, which preserves empty strings between separators, as well as at the start and end. + */ +template +C basicTokenizeString(std::basic_string_view s, std::basic_string_view separators); + +/** + * Like `basicTokenizeString` but specialized to the default `char` + */ +template +C tokenizeString(std::string_view s, std::string_view separators = " \t\n\r"); + +extern template std::list tokenizeString(std::string_view s, std::string_view separators); +extern template std::set tokenizeString(std::string_view s, std::string_view separators); +extern template std::vector tokenizeString(std::string_view s, std::string_view separators); + +/** + * Split a string, preserving empty strings between separators, as well as at the start and end. + * + * Returns a non-empty collection of strings. + */ +template +C basicSplitString(std::basic_string_view s, std::basic_string_view separators); +template +C splitString(std::string_view s, std::string_view separators); + +extern template std::list splitString(std::string_view s, std::string_view separators); +extern template std::set splitString(std::string_view s, std::string_view separators); +extern template std::vector splitString(std::string_view s, std::string_view separators); + /** * Concatenate the given strings with a separator between the elements. */ @@ -18,4 +50,20 @@ extern template std::string concatStringsSep(std::string_view, const std::list &); extern template std::string concatStringsSep(std::string_view, const std::vector &); +/** + * Ignore any empty strings at the start of the list, and then concatenate the + * given strings with a separator between the elements. + * + * @deprecated This function exists for historical reasons. You probably just + * want to use `concatStringsSep`. + */ +template +[[deprecated( + "Consider removing the empty string dropping behavior. If acceptable, use concatStringsSep instead.")]] std::string +dropEmptyInitThenConcatStringsSep(const std::string_view sep, const C & ss); + +extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::list &); +extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::set &); +extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::vector &); + } diff --git a/src/libutil/sync.hh b/src/libutil/sync.hh index 20dd6ee52..d340f3d97 100644 --- a/src/libutil/sync.hh +++ b/src/libutil/sync.hh @@ -7,6 +7,8 @@ #include #include +#include "error.hh" + namespace nix { /** @@ -47,7 +49,7 @@ public: friend SyncBase; Lock(SyncBase * s) : s(s), lk(s->mutex) { } public: - Lock(Lock && l) : s(l.s) { abort(); } + Lock(Lock && l) : s(l.s) { unreachable(); } Lock(const Lock & l) = delete; ~Lock() { } @@ -104,7 +106,7 @@ public: * Acquire read access to the inner value. When using * `std::shared_mutex`, this will use a shared lock. */ - ReadLock read() const { return ReadLock(const_cast(this)); } + ReadLock readLock() const { return ReadLock(const_cast(this)); } }; template diff --git a/src/libutil/unix/environment-variables.cc b/src/libutil/unix/environment-variables.cc index 9c6fd3b18..cd7c8f5e5 100644 --- a/src/libutil/unix/environment-variables.cc +++ b/src/libutil/unix/environment-variables.cc @@ -9,4 +9,14 @@ int setEnv(const char * name, const char * value) return ::setenv(name, value, 1); } +std::optional getEnvOs(const std::string & key) +{ + return getEnv(key); +} + +int setEnvOs(const OsString & name, const OsString & value) +{ + return setEnv(name.c_str(), value.c_str()); +} + } diff --git a/src/libutil/unix/file-path.cc b/src/libutil/unix/file-path.cc index 294048a2f..cccee86a1 100644 --- a/src/libutil/unix/file-path.cc +++ b/src/libutil/unix/file-path.cc @@ -8,16 +8,6 @@ namespace nix { -std::string os_string_to_string(PathViewNG::string_view path) -{ - return std::string { path }; -} - -std::filesystem::path::string_type string_to_os_string(std::string_view s) -{ - return std::string { s }; -} - std::optional maybePath(PathView path) { return { path }; diff --git a/src/libutil/unix/meson.build b/src/libutil/unix/meson.build index 38e5cd3aa..1c5bf27fb 100644 --- a/src/libutil/unix/meson.build +++ b/src/libutil/unix/meson.build @@ -4,6 +4,7 @@ sources += files( 'file-path.cc', 'file-system.cc', 'muxable-pipe.cc', + 'os-string.cc', 'processes.cc', 'signals.cc', 'users.cc', diff --git a/src/libutil/unix/monitor-fd.hh b/src/libutil/unix/monitor-fd.hh index 103894de9..b6610feff 100644 --- a/src/libutil/unix/monitor-fd.hh +++ b/src/libutil/unix/monitor-fd.hh @@ -40,7 +40,9 @@ public: #endif ; auto count = poll(fds, 1, -1); - if (count == -1) abort(); // can't happen + if (count == -1) + unreachable(); + /* This shouldn't happen, but can on macOS due to a bug. See rdar://37550628. diff --git a/src/libutil/unix/os-string.cc b/src/libutil/unix/os-string.cc new file mode 100644 index 000000000..8378afde2 --- /dev/null +++ b/src/libutil/unix/os-string.cc @@ -0,0 +1,21 @@ +#include +#include +#include +#include + +#include "file-path.hh" +#include "util.hh" + +namespace nix { + +std::string os_string_to_string(PathViewNG::string_view path) +{ + return std::string{path}; +} + +std::filesystem::path::string_type string_to_os_string(std::string_view s) +{ + return std::string{s}; +} + +} diff --git a/src/libutil/unix/processes.cc b/src/libutil/unix/processes.cc index 1af559a21..c5ce74acc 100644 --- a/src/libutil/unix/processes.cc +++ b/src/libutil/unix/processes.cc @@ -182,7 +182,7 @@ static pid_t doFork(bool allowVfork, ChildWrapperFunction & fun) #endif if (pid != 0) return pid; fun(); - abort(); + unreachable(); } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 698e181a1..db3ed1ddf 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1,5 +1,6 @@ #include "util.hh" #include "fmt.hh" +#include "file-path.hh" #include #include @@ -53,24 +54,6 @@ std::vector stringsToCharPtrs(const Strings & ss) ////////////////////////////////////////////////////////////////////// -template C tokenizeString(std::string_view s, std::string_view separators) -{ - C result; - auto pos = s.find_first_not_of(separators, 0); - while (pos != s.npos) { - auto end = s.find_first_of(separators, pos + 1); - if (end == s.npos) end = s.size(); - result.insert(result.end(), std::string(s, pos, end - pos)); - pos = s.find_first_not_of(separators, end); - } - return result; -} - -template Strings tokenizeString(std::string_view s, std::string_view separators); -template StringSet tokenizeString(std::string_view s, std::string_view separators); -template std::vector tokenizeString(std::string_view s, std::string_view separators); - - std::string chomp(std::string_view s) { size_t i = s.find_last_not_of(" \n\r\t"); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 877d15279..25128a900 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -28,49 +28,11 @@ std::vector stringsToCharPtrs(const Strings & ss); MakeError(FormatError, Error); -/** - * String tokenizer. - */ -template C tokenizeString(std::string_view s, std::string_view separators = " \t\n\r"); - - -/** - * Ignore any empty strings at the start of the list, and then concatenate the - * given strings with a separator between the elements. - * - * @deprecated This function exists for historical reasons. You probably just - * want to use `concatStringsSep`. - */ -template -[[deprecated("Consider removing the empty string dropping behavior. If acceptable, use concatStringsSep instead.")]] -std::string dropEmptyInitThenConcatStringsSep(const std::string_view sep, const C & ss) -{ - size_t size = 0; - - // TODO? remove to make sure we don't rely on the empty item ignoring behavior, - // or just get rid of this function by understanding the remaining calls. - // for (auto & i : ss) { - // // Make sure we don't rely on the empty item ignoring behavior - // assert(!i.empty()); - // break; - // } - - // need a cast to string_view since this is also called with Symbols - for (const auto & s : ss) size += sep.size() + std::string_view(s).size(); - std::string s; - s.reserve(size); - for (auto & i : ss) { - if (s.size() != 0) s += sep; - s += i; - } - return s; -} - -template -auto concatStrings(Parts && ... parts) +template +auto concatStrings(Parts &&... parts) -> std::enable_if_t<(... && std::is_convertible_v), std::string> { - std::string_view views[sizeof...(parts)] = { parts... }; + std::string_view views[sizeof...(parts)] = {parts...}; return concatStringsSep({}, views); } diff --git a/src/libutil/windows/environment-variables.cc b/src/libutil/windows/environment-variables.cc index 25ab9d63a..525d08c64 100644 --- a/src/libutil/windows/environment-variables.cc +++ b/src/libutil/windows/environment-variables.cc @@ -4,7 +4,30 @@ namespace nix { -int unsetenv(const char *name) +std::optional getEnvOs(const OsString & key) +{ + // Determine the required buffer size for the environment variable value + DWORD bufferSize = GetEnvironmentVariableW(key.c_str(), nullptr, 0); + if (bufferSize == 0) { + return std::nullopt; + } + + // Allocate a buffer to hold the environment variable value + std::wstring value{L'\0', bufferSize}; + + // Retrieve the environment variable value + DWORD resultSize = GetEnvironmentVariableW(key.c_str(), &value[0], bufferSize); + if (resultSize == 0) { + return std::nullopt; + } + + // Resize the string to remove the extra null characters + value.resize(resultSize); + + return value; +} + +int unsetenv(const char * name) { return -SetEnvironmentVariableA(name, nullptr); } @@ -14,4 +37,9 @@ int setEnv(const char * name, const char * value) return -SetEnvironmentVariableA(name, value); } +int setEnvOs(const OsString & name, const OsString & value) +{ + return -SetEnvironmentVariableW(name.c_str(), value.c_str()); +} + } diff --git a/src/libutil/windows/file-path.cc b/src/libutil/windows/file-path.cc index 3114ac4df..7405c426b 100644 --- a/src/libutil/windows/file-path.cc +++ b/src/libutil/windows/file-path.cc @@ -9,18 +9,6 @@ namespace nix { -std::string os_string_to_string(PathViewNG::string_view path) -{ - std::wstring_convert> converter; - return converter.to_bytes(std::filesystem::path::string_type { path }); -} - -std::filesystem::path::string_type string_to_os_string(std::string_view s) -{ - std::wstring_convert> converter; - return converter.from_bytes(std::string { s }); -} - std::optional maybePath(PathView path) { if (path.length() >= 3 && (('A' <= path[0] && path[0] <= 'Z') || ('a' <= path[0] && path[0] <= 'z')) && path[1] == ':' && WindowsPathTrait::isPathSep(path[2])) { diff --git a/src/libutil/windows/meson.build b/src/libutil/windows/meson.build index 00320877f..1c645fe05 100644 --- a/src/libutil/windows/meson.build +++ b/src/libutil/windows/meson.build @@ -4,6 +4,7 @@ sources += files( 'file-path.cc', 'file-system.cc', 'muxable-pipe.cc', + 'os-string.cc', 'processes.cc', 'users.cc', 'windows-async-pipe.cc', diff --git a/src/libutil/windows/os-string.cc b/src/libutil/windows/os-string.cc new file mode 100644 index 000000000..7507f9030 --- /dev/null +++ b/src/libutil/windows/os-string.cc @@ -0,0 +1,24 @@ +#include +#include +#include +#include + +#include "file-path.hh" +#include "file-path-impl.hh" +#include "util.hh" + +namespace nix { + +std::string os_string_to_string(PathViewNG::string_view path) +{ + std::wstring_convert> converter; + return converter.to_bytes(std::filesystem::path::string_type{path}); +} + +std::filesystem::path::string_type string_to_os_string(std::string_view s) +{ + std::wstring_convert> converter; + return converter.from_bytes(std::string{s}); +} + +} diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 9f7f557b5..878f3c828 100644 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -7,6 +7,7 @@ #include "eval-settings.hh" // for defexpr #include "users.hh" #include "tarball.hh" +#include "self-exe.hh" #include #include @@ -67,7 +68,7 @@ static void removeChannel(const std::string & name) channels.erase(name); writeChannels(); - runProgram(settings.nixBinDir + "/nix-env", true, { "--profile", profile, "--uninstall", name }); + runProgram(getNixBin("nix-env").string(), true, { "--profile", profile, "--uninstall", name }); } static Path nixDefExpr; @@ -118,7 +119,7 @@ static void update(const StringSet & channelNames) bool unpacked = false; if (std::regex_search(filename, std::regex("\\.tar\\.(gz|bz2|xz)$"))) { - runProgram(settings.nixBinDir + "/nix-build", false, { "--no-out-link", "--expr", "import " + unpackChannelPath + + runProgram(getNixBin("nix-build").string(), false, { "--no-out-link", "--expr", "import " + unpackChannelPath + "{ name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; }" }); unpacked = true; } @@ -143,7 +144,7 @@ static void update(const StringSet & channelNames) for (auto & expr : exprs) envArgs.push_back(std::move(expr)); envArgs.push_back("--quiet"); - runProgram(settings.nixBinDir + "/nix-env", false, envArgs); + runProgram(getNixBin("nix-env").string(), false, envArgs); // Make the channels appear in nix-env. struct stat st; @@ -244,7 +245,7 @@ static int main_nix_channel(int argc, char ** argv) case cListGenerations: if (!args.empty()) throw UsageError("'--list-generations' expects no arguments"); - std::cout << runProgram(settings.nixBinDir + "/nix-env", false, {"--profile", profile, "--list-generations"}) << std::flush; + std::cout << runProgram(getNixBin("nix-env").string(), false, {"--profile", profile, "--list-generations"}) << std::flush; break; case cRollback: if (args.size() > 1) @@ -256,7 +257,7 @@ static int main_nix_channel(int argc, char ** argv) } else { envArgs.push_back("--rollback"); } - runProgram(settings.nixBinDir + "/nix-env", false, envArgs); + runProgram(getNixBin("nix-env").string(), false, envArgs); break; } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 5e170c99d..ba2baccee 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -204,15 +204,15 @@ static void loadDerivations(EvalState & state, const SourcePath & nixExprPath, } -static long getPriority(EvalState & state, PackageInfo & drv) +static NixInt getPriority(EvalState & state, PackageInfo & drv) { - return drv.queryMetaInt("priority", 0); + return drv.queryMetaInt("priority", NixInt(0)); } -static long comparePriorities(EvalState & state, PackageInfo & drv1, PackageInfo & drv2) +static std::strong_ordering comparePriorities(EvalState & state, PackageInfo & drv1, PackageInfo & drv2) { - return getPriority(state, drv2) - getPriority(state, drv1); + return getPriority(state, drv2) <=> getPriority(state, drv1); } @@ -280,7 +280,7 @@ std::vector pickNewestOnly(EvalState & state, std::vector matches) auto & oneDrv = match.packageInfo; const auto drvName = DrvName { oneDrv.queryName() }; - long comparison = 1; + std::strong_ordering comparison = std::strong_ordering::greater; const auto itOther = newest.find(drvName.name); @@ -288,9 +288,9 @@ std::vector pickNewestOnly(EvalState & state, std::vector matches) auto & newestDrv = itOther->second.packageInfo; comparison = - oneDrv.querySystem() == newestDrv.querySystem() ? 0 : - oneDrv.querySystem() == settings.thisSystem ? 1 : - newestDrv.querySystem() == settings.thisSystem ? -1 : 0; + oneDrv.querySystem() == newestDrv.querySystem() ? std::strong_ordering::equal : + oneDrv.querySystem() == settings.thisSystem ? std::strong_ordering::greater : + newestDrv.querySystem() == settings.thisSystem ? std::strong_ordering::less : std::strong_ordering::equal; if (comparison == 0) comparison = comparePriorities(state, oneDrv, newestDrv); if (comparison == 0) @@ -625,13 +625,13 @@ static void upgradeDerivations(Globals & globals, continue; DrvName newName(j->queryName()); if (newName.name == drvName.name) { - int d = compareVersions(drvName.version, newName.version); + std::strong_ordering d = compareVersions(drvName.version, newName.version); if ((upgradeType == utLt && d < 0) || (upgradeType == utLeq && d <= 0) || (upgradeType == utEq && d == 0) || upgradeType == utAlways) { - long d2 = -1; + std::strong_ordering d2 = std::strong_ordering::less; if (bestElem != availElems.end()) { d2 = comparePriorities(*globals.state, *bestElem, *j); if (d2 == 0) d2 = compareVersions(bestVersion, newName.version); @@ -902,7 +902,7 @@ static VersionDiff compareVersionAgainstSet( for (auto & i : elems) { DrvName name2(i.queryName()); if (name.name == name2.name) { - int d = compareVersions(name.version, name2.version); + std::strong_ordering d = compareVersions(name.version, name2.version); if (d < 0) { diff = cvGreater; version = name2.version; @@ -1159,7 +1159,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) case cvEqual: ch = '='; break; case cvGreater: ch = '<'; break; case cvUnavail: ch = '-'; break; - default: abort(); + default: unreachable(); } if (xmlOutput) { diff --git a/src/nix-functional-tests b/src/nix-functional-tests new file mode 120000 index 000000000..ed0cdf60b --- /dev/null +++ b/src/nix-functional-tests @@ -0,0 +1 @@ +../tests/functional \ No newline at end of file diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index f073074e8..b4de42ba1 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -480,7 +480,7 @@ static void opQuery(Strings opFlags, Strings opArgs) } default: - abort(); + unreachable(); } } diff --git a/src/nix/config-check.cc b/src/nix/config-check.cc index 09d140733..1a6574de2 100644 --- a/src/nix/config-check.cc +++ b/src/nix/config-check.cc @@ -8,6 +8,7 @@ #include "store-api.hh" #include "local-fs-store.hh" #include "worker-protocol.hh" +#include "executable-path.hh" using namespace nix; @@ -39,6 +40,8 @@ void checkInfo(const std::string & msg) { } +namespace fs = std::filesystem; + struct CmdConfigCheck : StoreCommand { bool success = true; @@ -75,11 +78,13 @@ struct CmdConfigCheck : StoreCommand bool checkNixInPath() { - PathSet dirs; + std::set dirs; - for (auto & dir : tokenizeString(getEnv("PATH").value_or(""), ":")) - if (pathExists(dir + "/nix-env")) - dirs.insert(dirOf(canonPath(dir + "/nix-env", true))); + for (auto & dir : ExecutablePath::load().directories) { + auto candidate = dir / "nix-env"; + if (fs::exists(candidate)) + dirs.insert(fs::canonical(candidate).parent_path() ); + } if (dirs.size() != 1) { std::stringstream ss; @@ -94,18 +99,25 @@ struct CmdConfigCheck : StoreCommand bool checkProfileRoots(ref store) { - PathSet dirs; + std::set dirs; - for (auto & dir : tokenizeString(getEnv("PATH").value_or(""), ":")) { - Path profileDir = dirOf(dir); + for (auto & dir : ExecutablePath::load().directories) { + auto profileDir = dir.parent_path(); try { - Path userEnv = canonPath(profileDir, true); + auto userEnv = fs::weakly_canonical(profileDir); - if (store->isStorePath(userEnv) && hasSuffix(userEnv, "user-environment")) { - while (profileDir.find("/profiles/") == std::string::npos && std::filesystem::is_symlink(profileDir)) - profileDir = absPath(readLink(profileDir), dirOf(profileDir)); + auto noContainsProfiles = [&]{ + for (auto && part : profileDir) + if (part == "profiles") return false; + return true; + }; - if (profileDir.find("/profiles/") == std::string::npos) + if (store->isStorePath(userEnv.string()) && hasSuffix(userEnv.string(), "user-environment")) { + while (noContainsProfiles() && std::filesystem::is_symlink(profileDir)) + profileDir = fs::weakly_canonical( + profileDir.parent_path() / fs::read_symlink(profileDir)); + + if (noContainsProfiles()) dirs.insert(dir); } } catch (SystemError &) { diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 92ec3b78a..effc86a0a 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -415,7 +415,7 @@ struct Common : InstallableCommand, MixProfile if (buildEnvironment.providesStructuredAttrs()) { fixupStructuredAttrs( - PATHNG_LITERAL("sh"), + OS_STR("sh"), "NIX_ATTRS_SH_FILE", buildEnvironment.getAttrsSH(), rewrites, @@ -423,7 +423,7 @@ struct Common : InstallableCommand, MixProfile tmpDir ); fixupStructuredAttrs( - PATHNG_LITERAL("json"), + OS_STR("json"), "NIX_ATTRS_JSON_FILE", buildEnvironment.getAttrsJSON(), rewrites, @@ -447,7 +447,7 @@ struct Common : InstallableCommand, MixProfile const BuildEnvironment & buildEnvironment, const std::filesystem::path & tmpDir) { - auto targetFilePath = tmpDir / PATHNG_LITERAL(".attrs."); + auto targetFilePath = tmpDir / OS_STR(".attrs."); targetFilePath += ext; writeFile(targetFilePath.string(), content); diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index 46c94b211..2bc7fe82b 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -25,15 +25,17 @@ GroupedPaths getClosureInfo(ref store, const StorePath & toplevel) GroupedPaths groupedPaths; - for (auto & path : closure) { + for (auto const & path : closure) { /* Strip the output name. Unfortunately this is ambiguous (we can't distinguish between output names like "bin" and version suffixes like "unstable"). */ static std::regex regex("(.*)-([a-z]+|lib32|lib64)"); - std::smatch match; - std::string name(path.name()); + std::cmatch match; + std::string name{path.name()}; + std::string_view const origName = path.name(); std::string outputName; - if (std::regex_match(name, match, regex)) { + + if (std::regex_match(origName.begin(), origName.end(), match, regex)) { name = match[1]; outputName = match[2]; } diff --git a/src/nix/env.cc b/src/nix/env.cc index 26c168880..def2d48b5 100644 --- a/src/nix/env.cc +++ b/src/nix/env.cc @@ -5,6 +5,7 @@ #include "eval.hh" #include "run.hh" #include "strings.hh" +#include "executable-path.hh" using namespace nix; @@ -125,10 +126,11 @@ struct CmdShell : InstallablesCommand, MixEnvironment } } - auto unixPath = tokenizeString(getEnv("PATH").value_or(""), ":"); - unixPath.insert(unixPath.begin(), pathAdditions.begin(), pathAdditions.end()); - auto unixPathString = concatStringsSep(":", unixPath); - setEnv("PATH", unixPathString.c_str()); + // TODO: split losslessly; empty means . + auto unixPath = ExecutablePath::load(); + unixPath.directories.insert(unixPath.directories.begin(), pathAdditions.begin(), pathAdditions.end()); + auto unixPathString = unixPath.render(); + setEnvOs(OS_STR("PATH"), unixPathString.c_str()); for (auto const& pathV : extraPathVarMapping) { if (!extraPathVars[pathV.first].empty()) { diff --git a/src/nix/local.mk b/src/nix/local.mk index 28b30b586..b57f6b3e2 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -26,6 +26,8 @@ endif nix_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) $(INCLUDE_libflake) $(INCLUDE_libmain) -I src/libcmd -I doc/manual $(INCLUDE_nix) +nix_CXXFLAGS += -DNIX_BIN_DIR=\"$(NIX_ROOT)$(bindir)\" + nix_LIBS = libexpr libmain libfetchers libflake libstore libutil libcmd nix_LDFLAGS = $(THREAD_LDFLAGS) $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) $(LOWDOWN_LIBS) diff --git a/src/nix/main.cc b/src/nix/main.cc index 9d7d617cc..34de79ac8 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -19,6 +19,7 @@ #include "network-proxy.hh" #include "eval-cache.hh" #include "flake/flake.hh" +#include "self-exe.hh" #include #include @@ -366,6 +367,17 @@ void mainWrapped(int argc, char * * argv) initGC(); flake::initLib(flakeSettings); + /* Set the build hook location + + For builds we perform a self-invocation, so Nix has to be + self-aware. That is, it has to know where it is installed. We + don't think it's sentient. + */ + settings.buildHook.setDefault(Strings { + getNixBin({}).string(), + "__build-remote", + }); + #if __linux__ if (isRootUser()) { try { diff --git a/src/nix/meson.build b/src/nix/meson.build index 53bb083a9..798c98e33 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -20,6 +20,7 @@ deps_private_maybe_subproject = [ dependency('nix-util'), dependency('nix-store'), dependency('nix-expr'), + dependency('nix-flake'), dependency('nix-fetchers'), dependency('nix-main'), dependency('nix-cmd'), @@ -32,6 +33,21 @@ subdir('build-utils-meson/threads') subdir('build-utils-meson/export-all-symbols') +configdata = configuration_data() + +fs = import('fs') + +bindir = get_option('bindir') +if not fs.is_absolute(bindir) + bindir = get_option('prefix') / bindir +endif +configdata.set_quoted('NIX_BIN_DIR', bindir) + +config_h = configure_file( + configuration : configdata, + output : 'config-nix-cli.hh', +) + add_project_arguments( # TODO(Qyriad): Yes this is how the autoconf+Make system did it. # It would be nice for our headers to be idempotent instead. @@ -41,15 +57,17 @@ add_project_arguments( #'-include', 'config-fetchers.hh', '-include', 'config-main.hh', '-include', 'config-cmd.hh', + '-include', 'config-nix-cli.hh', language : 'cpp', ) subdir('build-utils-meson/diagnostics') subdir('build-utils-meson/generate-header') -nix_sources = files( +nix_sources = [config_h] + files( 'add-to-store.cc', 'app.cc', + 'self-exe.cc', 'build.cc', 'bundle.cc', 'cat.cc', @@ -174,3 +192,55 @@ this_exe = executable( link_args: linker_export_flags, install : true, ) + +meson.override_find_program('nix', this_exe) + +nix_symlinks = [ + 'nix-build', + 'nix-channel', + 'nix-collect-garbage', + 'nix-copy-closure', + 'nix-daemon', + 'nix-env', + 'nix-hash', + 'nix-instantiate', + 'nix-prefetch-url', + 'nix-shell', + 'nix-store', +] + +foreach linkname : nix_symlinks + install_symlink( + linkname, + # TODO(Qyriad): should these continue to be relative symlinks? + pointing_to : 'nix', + install_dir : get_option('bindir'), + # The 'runtime' tag is what executables default to, which we want to emulate here. + install_tag : 'runtime' + ) + t = custom_target( + command: ['ln', '-sf', fs.name(this_exe), '@OUTPUT@'], + output: linkname, + # TODO(Ericson2314): Don't do this once we have the `meson.override_find_program` working) + build_by_default: true + ) + # TODO(Ericson3214): Dosen't yet work + #meson.override_find_program(linkname, t) +endforeach + +install_symlink( + 'build-remote', + pointing_to : '..' / '..'/ get_option('bindir') / 'nix', + install_dir : get_option('libexecdir') / 'nix', + # The 'runtime' tag is what executables default to, which we want to emulate here. + install_tag : 'runtime' +) + +custom_target( + command: ['ln', '-sf', fs.name(this_exe), '@OUTPUT@'], + output: 'build-remote', + # TODO(Ericson2314): Don't do this once we have the `meson.override_find_program` working) + build_by_default: true +) +# TODO(Ericson3214): Dosen't yet work +#meson.override_find_program(linkname, t) diff --git a/src/nix/nix.md b/src/nix/nix.md index 4464bef37..443757a05 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -50,7 +50,7 @@ manual](https://nixos.org/manual/nix/stable/). > **Warning** \ > Installables are part of the unstable -> [`nix-command` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-nix-command), +> [`nix-command` experimental feature](@docroot@/development/experimental-features.md#xp-feature-nix-command), > and subject to change without notice. Many `nix` subcommands operate on one or more *installables*. @@ -59,9 +59,13 @@ These are command line arguments that represent something that can be realised i The following types of installable are supported by most commands: - [Flake output attribute](#flake-output-attribute) (experimental) + - This is the default - [Store path](#store-path) + - This is assumed if the argument is a Nix store path or a symlink to a Nix store path - [Nix file](#nix-file), optionally qualified by an attribute path + - Specified with `--file`/`-f` - [Nix expression](#nix-expression), optionally qualified by an attribute path + - Specified with `--expr` For most commands, if no installable is specified, `.` is assumed. That is, Nix will operate on the default flake output attribute of the flake in the current directory. @@ -70,9 +74,9 @@ That is, Nix will operate on the default flake output attribute of the flake in > **Warning** \ > Flake output attribute installables depend on both the -> [`flakes`](@docroot@/contributing/experimental-features.md#xp-feature-flakes) +> [`flakes`](@docroot@/development/experimental-features.md#xp-feature-flakes) > and -> [`nix-command`](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) +> [`nix-command`](@docroot@/development/experimental-features.md#xp-feature-nix-command) > experimental features, and subject to change without notice. Example: `nixpkgs#hello` @@ -178,9 +182,10 @@ that contains programs, and a `dev` output that provides development artifacts like C/C++ header files. The outputs on which `nix` commands operate are determined as follows: -* You can explicitly specify the desired outputs using the syntax - *installable*`^`*output1*`,`*...*`,`*outputN*. For example, you can - obtain the `dev` and `static` outputs of the `glibc` package: +* You can explicitly specify the desired outputs using the syntax *installable*`^`*output1*`,`*...*`,`*outputN* — that is, a caret followed immediately by a comma-separated list of derivation outputs to select. + For installables specified as [Flake output attributes](#flake-output-attribute) or [Store paths](#store-path), the output is specified in the same argument: + + For example, you can obtain the `dev` and `static` outputs of the `glibc` package: ```console # nix build 'nixpkgs#glibc^dev,static' @@ -195,6 +200,19 @@ operate are determined as follows: … ``` + For `--expr` and `-f`/`--file`, the derivation output is specified as part of the attribute path: + + ```console + $ nix build -f '' 'glibc^dev,static' + $ nix build --impure --expr 'import { }' 'glibc^dev,static' + ``` + + This syntax is the same even if the actual attribute path is empty: + + ```console + $ nix build --impure --expr 'let pkgs = import { }; in pkgs.glibc' '^dev,static' + ``` + * You can also specify that *all* outputs should be used using the syntax *installable*`^*`. For example, the following shows the size of all outputs of the `glibc` package in the binary cache: diff --git a/src/nix/package.nix b/src/nix/package.nix index fe83c6969..ef7265458 100644 --- a/src/nix/package.nix +++ b/src/nix/package.nix @@ -84,8 +84,6 @@ mkMesonDerivation (finalAttrs: { ] ); - outputs = [ "out" "dev" ]; - nativeBuildInputs = [ meson ninja @@ -114,12 +112,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/src/nix/repl.cc b/src/nix/repl.cc index a2f3e033e..5a570749f 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -1,12 +1,32 @@ #include "eval.hh" #include "eval-settings.hh" +#include "config-global.hh" #include "globals.hh" #include "command.hh" #include "installable-value.hh" #include "repl.hh" +#include "processes.hh" +#include "self-exe.hh" namespace nix { +void runNix(Path program, const Strings & args, + const std::optional & input = {}) +{ + auto subprocessEnv = getEnv(); + subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue(); + //isInteractive avoid grabling interactive commands + runProgram2(RunOptions { + .program = getNixBin(program).string(), + .args = args, + .environment = subprocessEnv, + .input = input, + .isInteractive = true, + }); + + return; +} + struct CmdRepl : RawInstallablesCommand { CmdRepl() { @@ -81,7 +101,8 @@ struct CmdRepl : RawInstallablesCommand lookupPath, openStore(), state, - getValues + getValues, + runNix ); repl->autoArgs = getAutoArgs(*repl->state); repl->initEnv(); diff --git a/src/nix/search.cc b/src/nix/search.cc index 78b811bec..60d3b0a84 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -10,6 +10,7 @@ #include "eval-cache.hh" #include "attr-path.hh" #include "hilite.hh" +#include "strings-inline.hh" #include #include diff --git a/src/nix/self-exe.cc b/src/nix/self-exe.cc new file mode 100644 index 000000000..a260bafd5 --- /dev/null +++ b/src/nix/self-exe.cc @@ -0,0 +1,38 @@ +#include "current-process.hh" +#include "file-system.hh" +#include "globals.hh" +#include "self-exe.hh" + +namespace nix { + +namespace fs = std::filesystem; + +fs::path getNixBin(std::optional binaryNameOpt) +{ + auto getBinaryName = [&] { return binaryNameOpt ? *binaryNameOpt : "nix"; }; + + // If the environment variable is set, use it unconditionally + if (auto envOpt = getEnvNonEmpty("NIX_BIN_DIR")) + return fs::path{*envOpt} / std::string{getBinaryName()}; + + // Use some-times avaiable OS tricks to get to the path of this Nix, and try that + if (auto selfOpt = getSelfExe()) { + fs::path path{*selfOpt}; + if (binaryNameOpt) + path = path.parent_path() / std::string{*binaryNameOpt}; + if (fs::exists(path)) + return path; + } + + // If `nix` exists at the hardcoded fallback path, use it. + { + auto path = fs::path{NIX_BIN_DIR} / std::string{getBinaryName()}; + if (fs::exists(path)) + return path; + } + + // return just the name, hoping the exe is on the `PATH` + return getBinaryName(); +} + +} diff --git a/src/nix/self-exe.hh b/src/nix/self-exe.hh new file mode 100644 index 000000000..0772afa67 --- /dev/null +++ b/src/nix/self-exe.hh @@ -0,0 +1,31 @@ +#pragma once +///@file + +#include + +namespace nix { + +/** + * Get a path to the given Nix binary. + * + * Normally, nix is installed according to `NIX_BIN_DIR`, which is set + * at compile time, but can be overridden. + * + * However, it may not have been installed at all. For example, if it's + * a static build, there's a good chance that it has been moved out of + * its installation directory. That makes `NIX_BIN_DIR` useless. + * Instead, we'll query the OS for the path to the current executable, + * using `getSelfExe()`. + * + * As a last resort, we resort to `PATH`. Hopefully we find a `nix` + * there that's compatible. If you're porting Nix to a new platform, + * that might be good enough for a while, but you'll want to improve + * `getSelfExe()` to work on your platform. + * + * @param binary_name the exact binary name we're looking up. Might be + * `nix-*` instead of `nix` for the legacy CLI commands. Optional to use + * current binary name. + */ +std::filesystem::path getNixBin(std::optional binary_name = {}); + +} diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 7b3357700..f54cc59d0 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -8,6 +8,8 @@ #include "attr-path.hh" #include "names.hh" #include "progress-bar.hh" +#include "executable-path.hh" +#include "self-exe.hh" using namespace nix; @@ -92,7 +94,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand { Activity act(*logger, lvlInfo, actUnknown, fmt("installing '%s' into profile '%s'...", store->printStorePath(storePath), profileDir)); - runProgram(settings.nixBinDir + "/nix-env", false, + runProgram(getNixBin("nix-env").string(), false, {"--profile", profileDir, "-i", store->printStorePath(storePath), "--no-sandbox"}); } @@ -102,23 +104,17 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand /* Return the profile in which Nix is installed. */ Path getProfileDir(ref store) { - Path where; - - for (auto & dir : tokenizeString(getEnv("PATH").value_or(""), ":")) - if (pathExists(dir + "/nix-env")) { - where = dir; - break; - } - - if (where == "") + auto whereOpt = ExecutablePath::load().findName(OS_STR("nix-env")); + if (!whereOpt) throw Error("couldn't figure out how Nix is installed, so I can't upgrade it"); + auto & where = *whereOpt; printInfo("found Nix in '%s'", where); - if (hasPrefix(where, "/run/current-system")) + if (hasPrefix(where.string(), "/run/current-system")) throw Error("Nix on NixOS must be upgraded via 'nixos-rebuild'"); - Path profileDir = dirOf(where); + Path profileDir = where.parent_path().string(); // Resolve profile to /nix/var/nix/profiles/ link. while (canonPath(profileDir).find("/profiles/") == std::string::npos && std::filesystem::is_symlink(profileDir)) @@ -128,7 +124,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand Path userEnv = canonPath(profileDir, true); - if (baseNameOf(where) != "bin" || + if (where.filename() != "bin" || !hasSuffix(userEnv, "user-environment")) throw Error("directory '%s' does not appear to be part of a Nix profile", where); diff --git a/src/perl/lib/Nix/Config.pm.in b/src/perl/lib/Nix/Config.pm.in index 508a15e15..ad51cff3b 100644 --- a/src/perl/lib/Nix/Config.pm.in +++ b/src/perl/lib/Nix/Config.pm.in @@ -5,7 +5,6 @@ use Nix::Store; $version = "@PACKAGE_VERSION@"; -$binDir = Nix::Store::getBinDir; $storeDir = Nix::Store::getStoreDir; %config = (); diff --git a/src/perl/lib/Nix/Store.pm b/src/perl/lib/Nix/Store.pm index 16f2e17c8..f2ae7e88f 100644 --- a/src/perl/lib/Nix/Store.pm +++ b/src/perl/lib/Nix/Store.pm @@ -24,7 +24,7 @@ our @EXPORT = qw( hashPath hashFile hashString convertHash signString checkSignature - getBinDir getStoreDir + getStoreDir setVerbosity ); diff --git a/src/perl/lib/Nix/Store.xs b/src/perl/lib/Nix/Store.xs index f951437c8..172c3500d 100644 --- a/src/perl/lib/Nix/Store.xs +++ b/src/perl/lib/Nix/Store.xs @@ -424,11 +424,6 @@ StoreWrapper::addTempRoot(char * storePath) } -SV * getBinDir() - PPCODE: - XPUSHs(sv_2mortal(newSVpv(settings.nixBinDir.c_str(), 0))); - - SV * getStoreDir() PPCODE: XPUSHs(sv_2mortal(newSVpv(settings.nixStore.c_str(), 0))); diff --git a/src/perl/meson.build b/src/perl/meson.build index 5fe7e1e28..dcb6a68a4 100644 --- a/src/perl/meson.build +++ b/src/perl/meson.build @@ -23,21 +23,22 @@ nix_perl_conf.set('PACKAGE_VERSION', meson.project_version()) # set error arguments #------------------------------------------------- error_args = [ - '-Werror=unused-result', '-Wdeprecated-copy', '-Wdeprecated-declarations', + '-Werror=suggest-override', + '-Werror=unused-result', '-Wignored-qualifiers', - '-Wno-pedantic', - '-Wno-non-virtual-dtor', - '-Wno-unused-parameter', - '-Wno-variadic-macros', - '-Wno-missing-field-initializers', - '-Wno-unknown-warning-option', - '-Wno-unused-variable', - '-Wno-literal-suffix', - '-Wno-reserved-user-defined-literal', '-Wno-duplicate-decl-specifier', + '-Wno-literal-suffix', + '-Wno-missing-field-initializers', + '-Wno-non-virtual-dtor', + '-Wno-pedantic', '-Wno-pointer-bool-conversion', + '-Wno-reserved-user-defined-literal', + '-Wno-unknown-warning-option', + '-Wno-unused-parameter', + '-Wno-unused-variable', + '-Wno-variadic-macros', ] add_project_arguments( diff --git a/src/perl/package.nix b/src/perl/package.nix index 26856e631..0b9343fba 100644 --- a/src/perl/package.nix +++ b/src/perl/package.nix @@ -73,5 +73,5 @@ perl.pkgs.toPerlModule (mkMesonDerivation (finalAttrs: { "--print-errorlogs" ]; - enableParallelBuilding = true; + strictDeps = false; })) diff --git a/tests/functional/.version b/tests/functional/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/tests/functional/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/tests/functional/ca/meson.build b/tests/functional/ca/meson.build new file mode 100644 index 000000000..f682ab28f --- /dev/null +++ b/tests/functional/ca/meson.build @@ -0,0 +1,33 @@ +configure_file( + input : 'config.nix.in', + output : 'config.nix', + configuration : test_confdata, +) + +suites += { + 'name': 'ca', + 'deps': [], + 'tests': [ + 'build-with-garbage-path.sh', + 'build.sh', + 'build-cache.sh', + 'concurrent-builds.sh', + 'derivation-json.sh', + 'duplicate-realisation-in-closure.sh', + 'eval-store.sh', + 'gc.sh', + 'import-derivation.sh', + 'new-build-cmd.sh', + 'nix-copy.sh', + 'nix-run.sh', + 'nix-shell.sh', + 'post-hook.sh', + 'recursive.sh', + 'repl.sh', + 'selfref-gc.sh', + 'signatures.sh', + 'substitute.sh', + 'why-depends.sh', + ], + 'workdir': meson.current_build_dir(), +} diff --git a/tests/functional/common.sh b/tests/functional/common.sh index d038aaf59..325fac44c 100644 --- a/tests/functional/common.sh +++ b/tests/functional/common.sh @@ -8,7 +8,8 @@ COMMON_SH_SOURCED=1 functionalTestsDir="$(readlink -f "$(dirname "${BASH_SOURCE[0]-$0}")")" -source "$functionalTestsDir/common/vars-and-functions.sh" +source "$functionalTestsDir/common/vars.sh" +source "$functionalTestsDir/common/functions.sh" source "$functionalTestsDir/common/init.sh" if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then diff --git a/tests/functional/common/vars-and-functions.sh b/tests/functional/common/functions.sh similarity index 64% rename from tests/functional/common/vars-and-functions.sh rename to tests/functional/common/functions.sh index 4316a30d5..d05fac4e7 100644 --- a/tests/functional/common/vars-and-functions.sh +++ b/tests/functional/common/functions.sh @@ -1,10 +1,10 @@ -# NOTE: instances of @variable@ are substituted as defined in /mk/templates.mk +# shellcheck shell=bash set -eu -o pipefail -if [[ -z "${COMMON_VARS_AND_FUNCTIONS_SH_SOURCED-}" ]]; then +if [[ -z "${COMMON_FUNCTIONS_SH_SOURCED-}" ]]; then -COMMON_VARS_AND_FUNCTIONS_SH_SOURCED=1 +COMMON_FUNCTIONS_SH_SOURCED=1 isTestOnNixOS() { [[ "${isTestOnNixOS:-}" == 1 ]] @@ -15,64 +15,14 @@ die() { exit 1 } -set +x - -commonDir="$(readlink -f "$(dirname "${BASH_SOURCE[0]-$0}")")" - -source "$commonDir/subst-vars.sh" -# Make sure shellcheck knows all these will be defined by the above generated snippet -: "${bindir?} ${coreutils?} ${dot?} ${SHELL?} ${PAGER?} ${busybox?} ${version?} ${system?} ${BUILD_SHARED_LIBS?}" - -source "$commonDir/paths.sh" -source "$commonDir/test-root.sh" - -test_nix_conf_dir=$TEST_ROOT/etc -test_nix_conf=$test_nix_conf_dir/nix.conf - -export TEST_HOME=$TEST_ROOT/test-home - -if ! isTestOnNixOS; then - export NIX_STORE_DIR - if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then - # Maybe the build directory is symlinked. - export NIX_IGNORE_SYMLINK_STORE=1 - NIX_STORE_DIR=$TEST_ROOT/store - fi - export NIX_LOCALSTATE_DIR=$TEST_ROOT/var - export NIX_LOG_DIR=$TEST_ROOT/var/log/nix - export NIX_STATE_DIR=$TEST_ROOT/var/nix - export NIX_CONF_DIR=$test_nix_conf_dir - export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/dSocket - unset NIX_USER_CONF_FILES - export _NIX_TEST_SHARED=$TEST_ROOT/shared - if [[ -n $NIX_STORE ]]; then - export _NIX_TEST_NO_SANDBOX=1 - fi - export _NIX_IN_TEST=$TEST_ROOT/shared - export _NIX_TEST_NO_LSOF=1 - export NIX_REMOTE=${NIX_REMOTE_-} - -fi # ! isTestOnNixOS - -unset NIX_PATH -export HOME=$TEST_HOME -unset XDG_STATE_HOME -unset XDG_DATA_HOME -unset XDG_CONFIG_HOME -unset XDG_CONFIG_DIRS -unset XDG_CACHE_HOME - -export IMPURE_VAR1=foo -export IMPURE_VAR2=bar - -cacheDir=$TEST_ROOT/binary-cache - readLink() { + # TODO fix this + # shellcheck disable=SC2012 ls -l "$1" | sed 's/.*->\ //' } clearProfiles() { - profiles="$HOME"/.local/state/nix/profiles + profiles="$HOME/.local/state/nix/profiles" rm -rf "$profiles" } @@ -105,11 +55,11 @@ doClearStore() { } clearCache() { - rm -rf "$cacheDir" + rm -rf "${cacheDir?}" } clearCacheCache() { - rm -f $TEST_HOME/.cache/nix/binary-cache* + rm -f "$TEST_HOME/.cache/nix/binary-cache"* } startDaemon() { @@ -122,7 +72,7 @@ startDaemon() { return fi # Start the daemon, wait for the socket to appear. - rm -f $NIX_DAEMON_SOCKET_PATH + rm -f "$NIX_DAEMON_SOCKET_PATH" PATH=$DAEMON_PATH nix --extra-experimental-features 'nix-command' daemon & _NIX_TEST_DAEMON_PID=$! export _NIX_TEST_DAEMON_PID @@ -151,14 +101,14 @@ killDaemon() { if [[ "${_NIX_TEST_DAEMON_PID-}" == '' ]]; then return fi - kill $_NIX_TEST_DAEMON_PID + kill "$_NIX_TEST_DAEMON_PID" for i in {0..100}; do - kill -0 $_NIX_TEST_DAEMON_PID 2> /dev/null || break + kill -0 "$_NIX_TEST_DAEMON_PID" 2> /dev/null || break sleep 0.1 done - kill -9 $_NIX_TEST_DAEMON_PID 2> /dev/null || true - wait $_NIX_TEST_DAEMON_PID || true - rm -f $NIX_DAEMON_SOCKET_PATH + kill -9 "$_NIX_TEST_DAEMON_PID" 2> /dev/null || true + wait "$_NIX_TEST_DAEMON_PID" || true + rm -f "$NIX_DAEMON_SOCKET_PATH" # Indicate daemon is stopped unset _NIX_TEST_DAEMON_PID # Restore old nix remote @@ -177,20 +127,17 @@ restartDaemon() { startDaemon } -if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then - _canUseSandbox=1 -fi - isDaemonNewer () { [[ -n "${NIX_DAEMON_PACKAGE:-}" ]] || return 0 local requiredVersion="$1" - local daemonVersion=$($NIX_DAEMON_PACKAGE/bin/nix daemon --version | cut -d' ' -f3) + local daemonVersion + daemonVersion=$("$NIX_DAEMON_PACKAGE/bin/nix" daemon --version | cut -d' ' -f3) [[ $(nix eval --expr "builtins.compareVersions ''$daemonVersion'' ''$requiredVersion''") -ge 0 ]] } skipTest () { echo "$1, skipping this test..." >&2 - exit 99 + exit 77 } TODO_NixOS() { @@ -236,7 +183,8 @@ expect() { expected="$1" shift "$@" && res=0 || res="$?" - if [[ $res -ne $expected ]]; then + # also match "negative" codes, which wrap around to >127 + if [[ $res -ne $expected && $res -ne $((256 + expected)) ]]; then echo "Expected exit code '$expected' but got '$res' from command ${*@Q}" >&2 return 1 fi @@ -250,7 +198,8 @@ expectStderr() { expected="$1" shift "$@" 2>&1 && res=0 || res="$?" - if [[ $res -ne $expected ]]; then + # also match "negative" codes, which wrap around to >127 + if [[ $res -ne $expected && $res -ne $((256 + expected)) ]]; then echo "Expected exit code '$expected' but got '$res' from command ${*@Q}" >&2 return 1 fi @@ -265,7 +214,7 @@ expectStderr() { # error: This error is expected # EOF assertStderr() { - diff -u /dev/stdin <($@ 2>/dev/null 2>&1) + diff -u /dev/stdin <("$@" 2>/dev/null 2>&1) } needLocalStore() { @@ -281,11 +230,9 @@ buggyNeedLocalStore() { enableFeatures() { local features="$1" - sed -i 's/experimental-features .*/& '"$features"'/' "$test_nix_conf_dir"/nix.conf + sed -i 's/experimental-features .*/& '"$features"'/' "${test_nix_conf?}" } -set -x - onError() { set +x echo "$0: test failed at:" >&2 @@ -295,13 +242,67 @@ onError() { done } +# Prints an error message prefix referring to the last call into this file. +# Ignores `expect` and `expectStderr` calls. +# Set a special exit code when test suite functions are misused, so that +# functions like expectStderr won't mistake them for expected Nix CLI errors. +# Suggestion: -101 (negative to indicate very abnormal, and beyond the normal +# range of signals) +# Example (showns as string): 'repl.sh:123: in call to grepQuiet: ' +# This function is inefficient, so it should only be used in error messages. +callerPrefix() { + # Find the closest caller that's not from this file + # using the bash `caller` builtin. + local i file line fn savedFn + # Use `caller` + for i in $(seq 0 100); do + caller "$i" > /dev/null || { + if [[ -n "${file:-}" ]]; then + echo "$file:$line: ${savedFn+in call to $savedFn: }" + fi + break + } + line="$(caller "$i" | cut -d' ' -f1)" + fn="$(caller "$i" | cut -d' ' -f2)" + file="$(caller "$i" | cut -d' ' -f3)" + if [[ $file != "${BASH_SOURCE[0]}" ]]; then + echo "$file:$line: ${savedFn+in call to $savedFn: }" + return + fi + case "$fn" in + # Ignore higher order functions that don't report any misuse of themselves + # This way a misuse of a foo in `expectStderr 1 foo` will be reported as + # calling foo, not expectStderr. + expect|expectStderr|callerPrefix) + ;; + *) + savedFn="$fn" + ;; + esac + done +} + +checkGrepArgs() { + local arg + for arg in "$@"; do + if [[ "$arg" != "${arg//$'\n'/_}" ]]; then + echo "$(callerPrefix)newline not allowed in arguments; grep would try each line individually as if connected by an OR operator" >&2 + return 155 # = -101 mod 256 + fi + done +} + # `grep -v` doesn't work well for exit codes. We want `!(exist line l. l # matches)`. It gives us `exist line l. !(l matches)`. # # `!` normally doesn't work well with `set -e`, but when we wrap in a # function it *does*. +# +# `command grep` lets us avoid re-checking the args by going directly to the +# executable. grepInverse() { - ! grep "$@" + checkGrepArgs "$@" && \ + ! command grep "$@" } # A shorthand, `> /dev/null` is a bit noisy. @@ -315,13 +316,26 @@ grepInverse() { # the closing of the pipe, the buffering of the pipe, and the speed of # the producer into the pipe. But rest assured we've seen it happen in # CI reliably. +# +# `command grep` lets us avoid re-checking the args by going directly to the +# executable. grepQuiet() { - grep "$@" > /dev/null + checkGrepArgs "$@" && \ + command grep "$@" > /dev/null } # The previous two, combined grepQuietInverse() { - ! grep "$@" > /dev/null + checkGrepArgs "$@" && \ + ! command grep "$@" > /dev/null +} + +# Wrap grep to remove its newline footgun; see checkGrepArgs. +# Note that we keep the checkGrepArgs calls in the other helpers, because some +# of them are negated and that would defeat this check. +grep() { + checkGrepArgs "$@" && \ + command grep "$@" } # Return the number of arguments @@ -331,4 +345,4 @@ count() { trap onError ERR -fi # COMMON_VARS_AND_FUNCTIONS_SH_SOURCED +fi # COMMON_FUNCTIONS_SH_SOURCED diff --git a/tests/functional/common/init.sh b/tests/functional/common/init.sh index d33ad5d57..d849c0734 100755 --- a/tests/functional/common/init.sh +++ b/tests/functional/common/init.sh @@ -7,10 +7,10 @@ if isTestOnNixOS; then mkdir -p "$test_nix_conf_dir" "$TEST_HOME" - export NIX_USER_CONF_FILES="$test_nix_conf_dir/nix.conf" + export NIX_USER_CONF_FILES="$test_nix_conf" mkdir -p "$test_nix_conf_dir" "$TEST_HOME" ! test -e "$test_nix_conf" - cat > "$test_nix_conf_dir/nix.conf" < "$test_nix_conf" < /dev/null); then + # Maybe the build directory is symlinked. + export NIX_IGNORE_SYMLINK_STORE=1 + NIX_STORE_DIR=$TEST_ROOT/store + fi + export NIX_LOCALSTATE_DIR=$TEST_ROOT/var + export NIX_LOG_DIR=$TEST_ROOT/var/log/nix + export NIX_STATE_DIR=$TEST_ROOT/var/nix + export NIX_CONF_DIR=$test_nix_conf_dir + export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/dSocket + unset NIX_USER_CONF_FILES + export _NIX_TEST_SHARED=$TEST_ROOT/shared + if [[ -n $NIX_STORE ]]; then + export _NIX_TEST_NO_SANDBOX=1 + fi + export _NIX_IN_TEST=$TEST_ROOT/shared + export _NIX_TEST_NO_LSOF=1 + export NIX_REMOTE=${NIX_REMOTE_-} + +fi # ! isTestOnNixOS + +unset NIX_PATH +export HOME=$TEST_HOME +unset XDG_STATE_HOME +unset XDG_DATA_HOME +unset XDG_CONFIG_HOME +unset XDG_CONFIG_DIRS +unset XDG_CACHE_HOME + +export IMPURE_VAR1=foo +export IMPURE_VAR2=bar + +# Used in other files +# shellcheck disable=SC2034 +cacheDir=$TEST_ROOT/binary-cache + +if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then + _canUseSandbox=1 +fi + +fi # COMMON_VARS_SH_SOURCED diff --git a/tests/functional/derivation-advanced-attributes.sh b/tests/functional/derivation-advanced-attributes.sh index 6c0c76b4c..271f17dc6 100755 --- a/tests/functional/derivation-advanced-attributes.sh +++ b/tests/functional/derivation-advanced-attributes.sh @@ -3,7 +3,7 @@ source common/test-root.sh source common/paths.sh -set -o pipefail +set -eu -o pipefail source characterisation/framework.sh diff --git a/tests/functional/dyn-drv/meson.build b/tests/functional/dyn-drv/meson.build new file mode 100644 index 000000000..3c671d013 --- /dev/null +++ b/tests/functional/dyn-drv/meson.build @@ -0,0 +1,19 @@ +configure_file( + input : 'config.nix.in', + output : 'config.nix', + configuration : test_confdata, +) + +suites += { + 'name': 'dyn-drv', + 'deps': [], + 'tests': [ + 'text-hashed-output.sh', + 'recursive-mod-json.sh', + 'build-built-drv.sh', + 'eval-outputOf.sh', + 'dep-built-drv.sh', + 'old-daemon-error-hack.sh', + ], + 'workdir': meson.current_build_dir(), +} diff --git a/tests/functional/flakes/local.mk b/tests/functional/flakes/local.mk new file mode 100644 index 000000000..71e50ad07 --- /dev/null +++ b/tests/functional/flakes/local.mk @@ -0,0 +1,24 @@ +flake-tests := \ + $(d)/flakes.sh \ + $(d)/develop.sh \ + $(d)/edit.sh \ + $(d)/run.sh \ + $(d)/mercurial.sh \ + $(d)/circular.sh \ + $(d)/init.sh \ + $(d)/inputs.sh \ + $(d)/follow-paths.sh \ + $(d)/bundle.sh \ + $(d)/check.sh \ + $(d)/unlocked-override.sh \ + $(d)/absolute-paths.sh \ + $(d)/absolute-attr-paths.sh \ + $(d)/build-paths.sh \ + $(d)/flake-in-submodule.sh \ + $(d)/prefetch.sh \ + $(d)/eval-cache.sh \ + $(d)/search-root.sh \ + $(d)/config.sh \ + $(d)/show.sh + +install-tests-groups += flake diff --git a/tests/functional/flakes/meson.build b/tests/functional/flakes/meson.build new file mode 100644 index 000000000..8c1afd6ff --- /dev/null +++ b/tests/functional/flakes/meson.build @@ -0,0 +1,28 @@ +suites += { + 'name': 'flakes', + 'deps': [], + 'tests': [ + 'flakes.sh', + 'develop.sh', + 'edit.sh', + 'run.sh', + 'mercurial.sh', + 'circular.sh', + 'init.sh', + 'inputs.sh', + 'follow-paths.sh', + 'bundle.sh', + 'check.sh', + 'unlocked-override.sh', + 'absolute-paths.sh', + 'absolute-attr-paths.sh', + 'build-paths.sh', + 'flake-in-submodule.sh', + 'prefetch.sh', + 'eval-cache.sh', + 'search-root.sh', + 'config.sh', + 'show.sh', + ], + 'workdir': meson.current_build_dir(), +} diff --git a/tests/functional/git-hashing/meson.build b/tests/functional/git-hashing/meson.build new file mode 100644 index 000000000..7486bfb8f --- /dev/null +++ b/tests/functional/git-hashing/meson.build @@ -0,0 +1,8 @@ +suites += { + 'name': 'git-hashing', + 'deps': [], + 'tests': [ + 'simple.sh', + ], + 'workdir': meson.current_build_dir(), +} diff --git a/tests/functional/lang.sh b/tests/functional/lang.sh index 8cb8e98fb..46cf3f1fe 100755 --- a/tests/functional/lang.sh +++ b/tests/functional/lang.sh @@ -50,11 +50,25 @@ set +x badDiff=0 badExitCode=0 +# Extra post-processing that's specific to each test case +postprocess() { + if [[ -e "lang/$1.postprocess" ]]; then + ( + # We could allow arbitrary interpreters in .postprocess, but that + # just exposes us to the complexity of not having /usr/bin/env in + # the sandbox. So let's just hardcode bash for now. + set -x; + bash "lang/$1.postprocess" "lang/$1" + ) + fi +} + for i in lang/parse-fail-*.nix; do echo "parsing $i (should fail)"; i=$(basename "$i" .nix) if expectStderr 1 nix-instantiate --parse - < "lang/$i.nix" > "lang/$i.err" then + postprocess "$i" diffAndAccept "$i" err err.exp else echo "FAIL: $i shouldn't parse" @@ -71,6 +85,7 @@ for i in lang/parse-okay-*.nix; do 2> "lang/$i.err" then sed "s!$(pwd)!/pwd!g" "lang/$i.out" "lang/$i.err" + postprocess "$i" diffAndAccept "$i" out exp diffAndAccept "$i" err err.exp else @@ -94,6 +109,7 @@ for i in lang/eval-fail-*.nix; do expectStderr 1 nix-instantiate $flags "lang/$i.nix" \ | sed "s!$(pwd)!/pwd!g" > "lang/$i.err" then + postprocess "$i" diffAndAccept "$i" err err.exp else echo "FAIL: $i shouldn't evaluate" @@ -109,6 +125,7 @@ for i in lang/eval-okay-*.nix; do if expect 0 nix-instantiate --eval --xml --no-location --strict \ "lang/$i.nix" > "lang/$i.out.xml" then + postprocess "$i" diffAndAccept "$i" out.xml exp.xml else echo "FAIL: $i should evaluate" @@ -129,6 +146,7 @@ for i in lang/eval-okay-*.nix; do 2> "lang/$i.err" then sed -i "s!$(pwd)!/pwd!g" "lang/$i.out" "lang/$i.err" + postprocess "$i" diffAndAccept "$i" out exp diffAndAccept "$i" err err.exp else diff --git a/tests/functional/lang/eval-fail-derivation-name.err.exp b/tests/functional/lang/eval-fail-derivation-name.err.exp index eb2206df1..0ef98674d 100644 --- a/tests/functional/lang/eval-fail-derivation-name.err.exp +++ b/tests/functional/lang/eval-fail-derivation-name.err.exp @@ -1,26 +1,26 @@ error: … while evaluating the attribute 'outPath' - at :19:9: - 18| value = commonAttrs // { - 19| outPath = builtins.getAttr outputName strict; + at ::: + | value = commonAttrs // { + | outPath = builtins.getAttr outputName strict; | ^ - 20| drvPath = strict.drvPath; + | drvPath = strict.drvPath; … while calling the 'getAttr' builtin - at :19:19: - 18| value = commonAttrs // { - 19| outPath = builtins.getAttr outputName strict; + at ::: + | value = commonAttrs // { + | outPath = builtins.getAttr outputName strict; | ^ - 20| drvPath = strict.drvPath; + | drvPath = strict.drvPath; … while calling the 'derivationStrict' builtin - at :9:12: - 8| - 9| strict = derivationStrict drvAttrs; + at ::: + | + | strict = derivationStrict drvAttrs; | ^ - 10| + | … while evaluating derivation '~jiggle~' - whose name attribute is located at /pwd/lang/eval-fail-derivation-name.nix:2:3 + whose name attribute is located at /pwd/lang/eval-fail-derivation-name.nix:: error: invalid derivation name: name '~jiggle~' contains illegal character '~'. Please pass a different 'name'. diff --git a/tests/functional/lang/eval-fail-derivation-name.postprocess b/tests/functional/lang/eval-fail-derivation-name.postprocess new file mode 100644 index 000000000..ffbc2b5d4 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivation-name.postprocess @@ -0,0 +1,9 @@ +# shellcheck shell=bash +set -euo pipefail +testcaseBasename=$1 + +# Line numbers change when derivation.nix docs are updated. +sed -i "$testcaseBasename.err" \ + -e 's/[0-9 ][0-9 ][0-9 ][0-9 ][0-9 ][0-9 ][0-9 ][0-9]\([^0-9]\)/\1/g' \ + -e 's/[0-9][0-9]*//g' \ + ; diff --git a/tests/functional/lang/eval-fail-fetchTree-negative.err.exp b/tests/functional/lang/eval-fail-fetchTree-negative.err.exp new file mode 100644 index 000000000..d9ba1f0b2 --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchTree-negative.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fetchTree' builtin + at /pwd/lang/eval-fail-fetchTree-negative.nix:1:1: + 1| builtins.fetchTree { + | ^ + 2| type = "file"; + + error: negative value given for fetchTree attr owner: -1 diff --git a/tests/functional/lang/eval-fail-fetchTree-negative.nix b/tests/functional/lang/eval-fail-fetchTree-negative.nix new file mode 100644 index 000000000..90bcab5d8 --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchTree-negative.nix @@ -0,0 +1,5 @@ +builtins.fetchTree { + type = "file"; + url = "file://eval-fail-fetchTree-negative.nix"; + owner = -1; +} diff --git a/tests/functional/lang/eval-fail-flake-ref-to-string-negative-integer.err.exp b/tests/functional/lang/eval-fail-flake-ref-to-string-negative-integer.err.exp new file mode 100644 index 000000000..25c8d7eaa --- /dev/null +++ b/tests/functional/lang/eval-fail-flake-ref-to-string-negative-integer.err.exp @@ -0,0 +1,14 @@ +error: + … while calling the 'seq' builtin + at /pwd/lang/eval-fail-flake-ref-to-string-negative-integer.nix:1:16: + 1| let n = -1; in builtins.seq n (builtins.flakeRefToString { + | ^ + 2| type = "github"; + + … while calling the 'flakeRefToString' builtin + at /pwd/lang/eval-fail-flake-ref-to-string-negative-integer.nix:1:32: + 1| let n = -1; in builtins.seq n (builtins.flakeRefToString { + | ^ + 2| type = "github"; + + error: negative value given for flake ref attr repo: -1 diff --git a/tests/functional/lang/eval-fail-flake-ref-to-string-negative-integer.nix b/tests/functional/lang/eval-fail-flake-ref-to-string-negative-integer.nix new file mode 100644 index 000000000..e0208eb25 --- /dev/null +++ b/tests/functional/lang/eval-fail-flake-ref-to-string-negative-integer.nix @@ -0,0 +1,7 @@ +let n = -1; in builtins.seq n (builtins.flakeRefToString { + type = "github"; + owner = "NixOS"; + repo = n; + ref = "23.05"; + dir = "lib"; +}) diff --git a/tests/functional/lang/eval-fail-fromJSON-overflowing.err.exp b/tests/functional/lang/eval-fail-fromJSON-overflowing.err.exp new file mode 100644 index 000000000..a39082b45 --- /dev/null +++ b/tests/functional/lang/eval-fail-fromJSON-overflowing.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fromJSON' builtin + at /pwd/lang/eval-fail-fromJSON-overflowing.nix:1:1: + 1| builtins.fromJSON ''{"attr": 18446744073709551615}'' + | ^ + 2| + + error: unsigned json number 18446744073709551615 outside of Nix integer range diff --git a/tests/functional/lang/eval-fail-fromJSON-overflowing.nix b/tests/functional/lang/eval-fail-fromJSON-overflowing.nix new file mode 100644 index 000000000..6dfbce3f6 --- /dev/null +++ b/tests/functional/lang/eval-fail-fromJSON-overflowing.nix @@ -0,0 +1 @@ +builtins.fromJSON ''{"attr": 18446744073709551615}'' diff --git a/tests/functional/lang/eval-fail-overflowing-add.err.exp b/tests/functional/lang/eval-fail-overflowing-add.err.exp new file mode 100644 index 000000000..6458cf1c9 --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-add.err.exp @@ -0,0 +1,6 @@ +error: integer overflow in adding 9223372036854775807 + 1 + at /pwd/lang/eval-fail-overflowing-add.nix:4:8: + 3| b = 1; + 4| in a + b + | ^ + 5| diff --git a/tests/functional/lang/eval-fail-overflowing-add.nix b/tests/functional/lang/eval-fail-overflowing-add.nix new file mode 100644 index 000000000..24258fc20 --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-add.nix @@ -0,0 +1,4 @@ +let + a = 9223372036854775807; + b = 1; +in a + b diff --git a/tests/functional/lang/eval-fail-overflowing-div.err.exp b/tests/functional/lang/eval-fail-overflowing-div.err.exp new file mode 100644 index 000000000..8ce07d4d6 --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-div.err.exp @@ -0,0 +1,23 @@ +error: + … while calling the 'seq' builtin + at /pwd/lang/eval-fail-overflowing-div.nix:7:4: + 6| b = -1; + 7| in builtins.seq intMin (builtins.seq b (intMin / b)) + | ^ + 8| + + … while calling the 'seq' builtin + at /pwd/lang/eval-fail-overflowing-div.nix:7:25: + 6| b = -1; + 7| in builtins.seq intMin (builtins.seq b (intMin / b)) + | ^ + 8| + + … while calling the 'div' builtin + at /pwd/lang/eval-fail-overflowing-div.nix:7:48: + 6| b = -1; + 7| in builtins.seq intMin (builtins.seq b (intMin / b)) + | ^ + 8| + + error: integer overflow in dividing -9223372036854775808 / -1 diff --git a/tests/functional/lang/eval-fail-overflowing-div.nix b/tests/functional/lang/eval-fail-overflowing-div.nix new file mode 100644 index 000000000..44fbe9d7e --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-div.nix @@ -0,0 +1,7 @@ +let + # lol, this has to be written as an expression like this because negative + # numbers use unary negation rather than parsing directly, and 2**63 is out + # of range + intMin = -9223372036854775807 - 1; + b = -1; +in builtins.seq intMin (builtins.seq b (intMin / b)) diff --git a/tests/functional/lang/eval-fail-overflowing-mul.err.exp b/tests/functional/lang/eval-fail-overflowing-mul.err.exp new file mode 100644 index 000000000..f42b39d4d --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-mul.err.exp @@ -0,0 +1,16 @@ +error: + … while calling the 'mul' builtin + at /pwd/lang/eval-fail-overflowing-mul.nix:3:10: + 2| a = 4294967297; + 3| in a * a * a + | ^ + 4| + + … while calling the 'mul' builtin + at /pwd/lang/eval-fail-overflowing-mul.nix:3:6: + 2| a = 4294967297; + 3| in a * a * a + | ^ + 4| + + error: integer overflow in multiplying 4294967297 * 4294967297 diff --git a/tests/functional/lang/eval-fail-overflowing-mul.nix b/tests/functional/lang/eval-fail-overflowing-mul.nix new file mode 100644 index 000000000..6081d9c7b --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-mul.nix @@ -0,0 +1,3 @@ +let + a = 4294967297; +in a * a * a diff --git a/tests/functional/lang/eval-fail-overflowing-sub.err.exp b/tests/functional/lang/eval-fail-overflowing-sub.err.exp new file mode 100644 index 000000000..66a3a03f8 --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-sub.err.exp @@ -0,0 +1,9 @@ +error: + … while calling the 'sub' builtin + at /pwd/lang/eval-fail-overflowing-sub.nix:4:6: + 3| b = 2; + 4| in a - b + | ^ + 5| + + error: integer overflow in subtracting -9223372036854775807 - 2 diff --git a/tests/functional/lang/eval-fail-overflowing-sub.nix b/tests/functional/lang/eval-fail-overflowing-sub.nix new file mode 100644 index 000000000..229b8c6d2 --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-sub.nix @@ -0,0 +1,4 @@ +let + a = -9223372036854775807; + b = 2; +in a - b diff --git a/tests/functional/lang/eval-fail-pipe-operators.err.exp b/tests/functional/lang/eval-fail-pipe-operators.err.exp new file mode 100644 index 000000000..49f3fa8ad --- /dev/null +++ b/tests/functional/lang/eval-fail-pipe-operators.err.exp @@ -0,0 +1,5 @@ +error: experimental Nix feature 'pipe-operators' is disabled; add '--extra-experimental-features pipe-operators' to enable it + at /pwd/lang/eval-fail-pipe-operators.nix:1:3: + 1| 1 |> 2 + | ^ + 2| diff --git a/tests/functional/lang/eval-fail-pipe-operators.nix b/tests/functional/lang/eval-fail-pipe-operators.nix new file mode 100644 index 000000000..433e0fd7f --- /dev/null +++ b/tests/functional/lang/eval-fail-pipe-operators.nix @@ -0,0 +1 @@ +1 |> 2 diff --git a/tests/functional/lang/parse-fail-undef-var-2.err.exp b/tests/functional/lang/parse-fail-undef-var-2.err.exp index 393c454dd..96e87b2aa 100644 --- a/tests/functional/lang/parse-fail-undef-var-2.err.exp +++ b/tests/functional/lang/parse-fail-undef-var-2.err.exp @@ -1,4 +1,4 @@ -error: syntax error, unexpected ':', expecting '}' +error: syntax error, unexpected ':', expecting '}' or ',' at «stdin»:3:13: 2| 3| f = {x, y : ["baz" "bar" z "bat"]}: x + y; diff --git a/tests/functional/local-overlay-store/common.sh b/tests/functional/local-overlay-store/common.sh index b171f91f4..27338ea23 100644 --- a/tests/functional/local-overlay-store/common.sh +++ b/tests/functional/local-overlay-store/common.sh @@ -1,4 +1,5 @@ -source ../common/vars-and-functions.sh +source ../common/vars.sh +source ../common/functions.sh TODO_NixOS diff --git a/tests/functional/local-overlay-store/meson.build b/tests/functional/local-overlay-store/meson.build new file mode 100644 index 000000000..6ff5d3169 --- /dev/null +++ b/tests/functional/local-overlay-store/meson.build @@ -0,0 +1,18 @@ +suites += { + 'name': 'local-overlay-store', + 'deps': [], + 'tests': [ + 'check-post-init.sh', + 'redundant-add.sh', + 'build.sh', + 'bad-uris.sh', + 'add-lower.sh', + 'delete-refs.sh', + 'delete-duplicate.sh', + 'gc.sh', + 'verify.sh', + 'optimise.sh', + 'stale-file-handle.sh', + ], + 'workdir': meson.current_build_dir(), +} diff --git a/tests/functional/local.mk b/tests/functional/local.mk index 797002e92..8b4945cac 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -1,23 +1,5 @@ nix_tests = \ test-infra.sh \ - flakes/flakes.sh \ - flakes/develop.sh \ - flakes/edit.sh \ - flakes/run.sh \ - flakes/mercurial.sh \ - flakes/circular.sh \ - flakes/init.sh \ - flakes/inputs.sh \ - flakes/follow-paths.sh \ - flakes/bundle.sh \ - flakes/check.sh \ - flakes/unlocked-override.sh \ - flakes/absolute-paths.sh \ - flakes/absolute-attr-paths.sh \ - flakes/build-paths.sh \ - flakes/flake-in-submodule.sh \ - flakes/prefetch.sh \ - flakes/eval-cache.sh \ gc.sh \ nix-collect-garbage-d.sh \ remote-store.sh \ @@ -61,7 +43,6 @@ nix_tests = \ restricted.sh \ fetchGitSubmodules.sh \ fetchGitVerification.sh \ - flakes/search-root.sh \ readfile-context.sh \ nix-channel.sh \ recursive.sh \ @@ -102,7 +83,6 @@ nix_tests = \ nix-copy-ssh-ng.sh \ post-hook.sh \ function-trace.sh \ - flakes/config.sh \ fmt.sh \ eval-store.sh \ why-depends.sh \ @@ -125,7 +105,6 @@ nix_tests = \ store-info.sh \ fetchClosure.sh \ completions.sh \ - flakes/show.sh \ impure-derivations.sh \ path-from-hash-part.sh \ path-info.sh \ diff --git a/tests/functional/meson.build b/tests/functional/meson.build new file mode 100644 index 000000000..ebecdd9e8 --- /dev/null +++ b/tests/functional/meson.build @@ -0,0 +1,266 @@ +project('nix-functional-tests', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.3', + license : 'LGPL-2.1-or-later', +) + +fs = import('fs') + +# Need to combine source and build trees +run_command( + 'rsync', + '-a', + '--copy-unsafe-links', + meson.current_source_dir() / '', + meson.current_build_dir() / '', +) +# This current-source-escaping relative is no good because we don't know +# where the build directory will be, therefore we fix it up. Once the +# Make build system is gone, we should think about doing this better. +scripts_dir = fs.relative_to( + meson.current_source_dir() / '..' / '..' / 'scripts', + meson.current_build_dir(), +) +run_command( + 'sed', + '-i', meson.current_build_dir() / 'bash-profile.sh', + '-e', 's^../../scripts^@0@^'.format(scripts_dir), +) + +nix = find_program('nix') +bash = find_program('bash', native : true) +busybox = find_program('busybox', native : true, required : false) +coreutils = find_program('coreutils', native : true) +dot = find_program('dot', native : true, required : false) + +nix_bin_dir = fs.parent(nix.full_path()) + +test_confdata = { + 'bindir': nix_bin_dir, + 'coreutils': fs.parent(coreutils.full_path()), + 'dot': dot.found() ? dot.full_path() : '', + 'bash': bash.full_path(), + 'sandbox_shell': busybox.found() ? busybox.full_path() : '', + 'PACKAGE_VERSION': meson.project_version(), + 'system': host_machine.cpu_family() + '-' + host_machine.system(), +} + +# Just configures `common/vars-and-functions.sh.in`. +# Done as a subdir() so Meson places it under `common` in the build directory as well. +subdir('common') + +config_nix_in = configure_file( + input : 'config.nix.in', + output : 'config.nix', + configuration : test_confdata, +) + +suites = [ + { + 'name' : 'main', + 'deps': [], + 'tests': [ + 'test-infra.sh', + 'gc.sh', + 'nix-collect-garbage-d.sh', + 'remote-store.sh', + 'legacy-ssh-store.sh', + 'lang.sh', + 'lang-gc.sh', + 'characterisation-test-infra.sh', + 'experimental-features.sh', + 'fetchMercurial.sh', + 'gc-auto.sh', + 'user-envs.sh', + 'user-envs-migration.sh', + 'binary-cache.sh', + 'multiple-outputs.sh', + 'nix-build.sh', + 'gc-concurrent.sh', + 'repair.sh', + 'fixed.sh', + 'export-graph.sh', + 'timeout.sh', + 'fetchGitRefs.sh', + 'gc-runtime.sh', + 'tarball.sh', + 'fetchGit.sh', + 'fetchurl.sh', + 'fetchPath.sh', + 'fetchTree-file.sh', + 'simple.sh', + 'referrers.sh', + 'optimise-store.sh', + 'substitute-with-invalid-ca.sh', + 'signing.sh', + 'hash-convert.sh', + 'hash-path.sh', + 'gc-non-blocking.sh', + 'check.sh', + 'nix-shell.sh', + 'check-refs.sh', + 'build-remote-input-addressed.sh', + 'secure-drv-outputs.sh', + 'restricted.sh', + 'fetchGitSubmodules.sh', + 'fetchGitVerification.sh', + 'readfile-context.sh', + 'nix-channel.sh', + 'recursive.sh', + 'dependencies.sh', + 'check-reqs.sh', + 'build-remote-content-addressed-fixed.sh', + 'build-remote-content-addressed-floating.sh', + 'build-remote-trustless-should-pass-0.sh', + 'build-remote-trustless-should-pass-1.sh', + 'build-remote-trustless-should-pass-2.sh', + 'build-remote-trustless-should-pass-3.sh', + 'build-remote-trustless-should-fail-0.sh', + 'build-remote-with-mounted-ssh-ng.sh', + 'nar-access.sh', + 'impure-eval.sh', + 'pure-eval.sh', + 'eval.sh', + 'repl.sh', + 'binary-cache-build-remote.sh', + 'search.sh', + 'logging.sh', + 'export.sh', + 'config.sh', + 'add.sh', + 'chroot-store.sh', + 'filter-source.sh', + 'misc.sh', + 'dump-db.sh', + 'linux-sandbox.sh', + 'supplementary-groups.sh', + 'build-dry.sh', + 'structured-attrs.sh', + 'shell.sh', + 'brotli.sh', + 'zstd.sh', + 'compression-levels.sh', + 'nix-copy-ssh.sh', + 'nix-copy-ssh-ng.sh', + 'post-hook.sh', + 'function-trace.sh', + 'fmt.sh', + 'eval-store.sh', + 'why-depends.sh', + 'derivation-json.sh', + 'derivation-advanced-attributes.sh', + 'import-derivation.sh', + 'nix_path.sh', + 'case-hack.sh', + 'placeholders.sh', + 'ssh-relay.sh', + 'build.sh', + 'build-delete.sh', + 'output-normalization.sh', + 'selfref-gc.sh', + 'db-migration.sh', + 'bash-profile.sh', + 'pass-as-file.sh', + 'nix-profile.sh', + 'suggestions.sh', + 'store-info.sh', + 'fetchClosure.sh', + 'completions.sh', + 'impure-derivations.sh', + 'path-from-hash-part.sh', + 'path-info.sh', + 'toString-path.sh', + 'read-only-store.sh', + 'nested-sandboxing.sh', + 'impure-env.sh', + 'debugger.sh', + 'extra-sandbox-profile.sh', + 'help.sh', + ], + 'workdir': meson.current_build_dir(), + }, +] + +nix_store = dependency('nix-store', required : false) +if nix_store.found() + subdir('test-libstoreconsumer') + suites += { + 'name': 'libstoreconsumer', + 'deps': [ + libstoreconsumer_tester, + ], + 'tests': [ + 'test-libstoreconsumer.sh', + ], + 'workdir': meson.current_build_dir(), + } + +endif + +# Plugin tests require shared libraries support. +nix_expr = dependency('nix-expr', required : false) +if nix_expr.found() and get_option('default_library') != 'static' + subdir('plugins') + suites += { + 'name': 'plugins', + 'deps': [ + libplugintest, + ], + 'tests': [ + 'plugins.sh', + ], + 'workdir': meson.current_build_dir(), + } +endif + +subdir('ca') +subdir('dyn-drv') +subdir('flakes') +subdir('git-hashing') +subdir('local-overlay-store') + +foreach suite : suites + foreach script : suite['tests'] + workdir = suite['workdir'] + prefix = fs.relative_to(workdir, meson.project_build_root()) + + script = script + # Turns, e.g., `tests/functional/flakes/show.sh` into a Meson test target called + # `functional-flakes-show`. + name = fs.replace_suffix(prefix / script, '') + + test( + name, + bash, + args: [ + '-x', + '-e', + '-u', + '-o', 'pipefail', + script, + ], + suite : suite['name'], + env : { + 'TEST_NAME': name, + 'NIX_REMOTE': '', + 'PS4': '+(${BASH_SOURCE[0]-$0}:$LINENO) ', + }, + # some tests take 15+ seconds even on an otherwise idle machine, on a loaded machine + # this can easily drive them to failure. give them more time than default of 30sec + timeout : 300, + # Used for target dependency/ordering tracking, not adding compiler flags or anything. + depends : suite['deps'], + workdir : workdir, + # Won't pass until man pages are generated + should_fail : suite['name'] == 'main' and script == 'help.sh' + ) + endforeach +endforeach diff --git a/tests/functional/nested-sandboxing/command.sh b/tests/functional/nested-sandboxing/command.sh index 69366486c..e9c40a5d9 100644 --- a/tests/functional/nested-sandboxing/command.sh +++ b/tests/functional/nested-sandboxing/command.sh @@ -1,3 +1,5 @@ +set -eu -o pipefail + export NIX_BIN_DIR=$(dirname $(type -p nix)) # TODO Get Nix and its closure more flexibly export EXTRA_SANDBOX="/nix/store $(dirname $NIX_BIN_DIR)" diff --git a/tests/functional/nested-sandboxing/runner.nix b/tests/functional/nested-sandboxing/runner.nix index 9a5822c88..1e79d5065 100644 --- a/tests/functional/nested-sandboxing/runner.nix +++ b/tests/functional/nested-sandboxing/runner.nix @@ -6,7 +6,10 @@ mkDerivation { name = "nested-sandboxing"; busybox = builtins.getEnv "busybox"; EXTRA_SANDBOX = builtins.getEnv "EXTRA_SANDBOX"; - buildCommand = if altitude == 0 then '' + buildCommand = '' + set -x + set -eu -o pipefail + '' + (if altitude == 0 then '' echo Deep enough! > $out '' else '' cp -r ${../common} ./common @@ -20,5 +23,5 @@ mkDerivation { source ./nested-sandboxing/command.sh runNixBuild ${storeFun} ${toString altitude} >> $out - ''; + ''); } diff --git a/tests/functional/nix_path.sh b/tests/functional/nix_path.sh index e6a2193f3..90cba1f0c 100755 --- a/tests/functional/nix_path.sh +++ b/tests/functional/nix_path.sh @@ -14,3 +14,75 @@ nix-instantiate --eval -E '' --restrict-eval [[ $(nix-instantiate --find-file by-absolute-path/simple.nix) = $PWD/simple.nix ]] [[ $(nix-instantiate --find-file by-relative-path/simple.nix) = $PWD/simple.nix ]] + +# this is the human-readable specification for the following test cases of interactions between various ways of specifying NIX_PATH. +# TODO: the actual tests are incomplete and too manual. +# there should be 43 of them, since the table has 9 rows and columns, and 2 interactions are meaningless +# ideally they would work off the table programmatically. +# +# | precedence | hard-coded | nix-path in file | extra-nix-path in file | nix-path in env | extra-nix-path in env | NIX_PATH | nix-path | extra-nix-path | -I | +# |------------------------|------------|------------------|------------------------|-----------------|-----------------------|-----------|-----------|-----------------|-----------------| +# | hard-coded | x | ^override | ^append | ^override | ^append | ^override | ^override | ^append | ^prepend | +# | nix-path in file | | last wins | ^append | ^override | ^append | ^override | ^override | ^append | ^prepend | +# | extra-nix-path in file | | | append in order | ^override | ^append | ^override | ^override | ^append | ^prepend | +# | nix-path in env | | | | last wins | ^append | ^override | ^override | ^append | ^prepend | +# | extra-nix-path in env | | | | | append in order | ^override | ^override | ^append | ^prepend | +# | NIX_PATH | | | | | | x | ^override | ^append | ^prepend | +# | nix-path | | | | | | | last wins | ^append | ^prepend | +# | extra-nix-path | | | | | | | | append in order | append in order | +# | -I | | | | | | | | | append in order | + +unset NIX_PATH + +mkdir -p $TEST_ROOT/{from-nix-path-file,from-NIX_PATH,from-nix-path,from-extra-nix-path,from-I} +for i in from-nix-path-file from-NIX_PATH from-nix-path from-extra-nix-path from-I; do + touch $TEST_ROOT/$i/only-$i.nix +done + +# finding something that's not in any of the default paths fails +( ! $(nix-instantiate --find-file test) ) + +echo "nix-path = test=$TEST_ROOT/from-nix-path-file" >> "$test_nix_conf" + +# Use nix.conf in absence of NIX_PATH +[[ $(nix-instantiate --find-file test) = $TEST_ROOT/from-nix-path-file ]] + +# NIX_PATH overrides nix.conf +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --find-file test) = $TEST_ROOT/from-NIX_PATH ]] +# if NIX_PATH does not have the desired entry, it fails +(! NIX_PATH=test=$TEST_ROOT nix-instantiate --find-file test/only-from-nix-path-file.nix) + +# -I extends nix.conf +[[ $(nix-instantiate -I test=$TEST_ROOT/from-I --find-file test/only-from-I.nix) = $TEST_ROOT/from-I/only-from-I.nix ]] +# if -I does not have the desired entry, the value from nix.conf is used +[[ $(nix-instantiate -I test=$TEST_ROOT/from-I --find-file test/only-from-nix-path-file.nix) = $TEST_ROOT/from-nix-path-file/only-from-nix-path-file.nix ]] + +# -I extends NIX_PATH +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate -I test=$TEST_ROOT/from-I --find-file test/only-from-I.nix) = $TEST_ROOT/from-I/only-from-I.nix ]] +# -I takes precedence over NIX_PATH +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate -I test=$TEST_ROOT/from-I --find-file test) = $TEST_ROOT/from-I ]] +# if -I does not have the desired entry, the value from NIX_PATH is used +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate -I test=$TEST_ROOT/from-I --find-file test/only-from-NIX_PATH.nix) = $TEST_ROOT/from-NIX_PATH/only-from-NIX_PATH.nix ]] + +# --extra-nix-path extends NIX_PATH +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --extra-nix-path test=$TEST_ROOT/from-extra-nix-path --find-file test/only-from-extra-nix-path.nix) = $TEST_ROOT/from-extra-nix-path/only-from-extra-nix-path.nix ]] +# if --extra-nix-path does not have the desired entry, the value from NIX_PATH is used +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --extra-nix-path test=$TEST_ROOT/from-extra-nix-path --find-file test/only-from-NIX_PATH.nix) = $TEST_ROOT/from-NIX_PATH/only-from-NIX_PATH.nix ]] + +# --nix-path overrides NIX_PATH +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path --find-file test) = $TEST_ROOT/from-nix-path ]] +# if --nix-path does not have the desired entry, it fails +(! NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path --find-file test/only-from-NIX_PATH.nix) + +# --nix-path overrides nix.conf +[[ $(nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path --find-file test) = $TEST_ROOT/from-nix-path ]] +(! nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path --find-file test/only-from-nix-path-file.nix) + +# --extra-nix-path extends nix.conf +[[ $(nix-instantiate --extra-nix-path test=$TEST_ROOT/from-extra-nix-path --find-file test/only-from-extra-nix-path.nix) = $TEST_ROOT/from-extra-nix-path/only-from-extra-nix-path.nix ]] +# if --extra-nix-path does not have the desired entry, it is taken from nix.conf +[[ $(nix-instantiate --extra-nix-path test=$TEST_ROOT/from-extra-nix-path --find-file test) = $TEST_ROOT/from-nix-path-file ]] + +# -I extends --nix-path +[[ $(nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path -I test=$TEST_ROOT/from-I --find-file test/only-from-I.nix) = $TEST_ROOT/from-I/only-from-I.nix ]] +[[ $(nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path -I test=$TEST_ROOT/from-I --find-file test/only-from-nix-path.nix) = $TEST_ROOT/from-nix-path/only-from-nix-path.nix ]] diff --git a/tests/functional/package.nix b/tests/functional/package.nix new file mode 100644 index 000000000..277711123 --- /dev/null +++ b/tests/functional/package.nix @@ -0,0 +1,117 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config +, rsync + +, jq +, git +, mercurial +, util-linux + +, nix-store +, nix-expr +, nix-cli + +, rapidcheck +, gtest +, runCommand + +, busybox-sandbox-shell ? null + +# Configuration Options + +, version + +# For running the functional tests against a different pre-built Nix. +, test-daemon ? null +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-functional-tests"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../scripts/nix-profile.sh.in + ../../.version + ../../tests/functional + ./. + ]; + + # Hack for sake of the dev shell + passthru.baseNativeBuildInputs = [ + meson + ninja + pkg-config + rsync + + jq + git + mercurial + ] ++ lib.optionals stdenv.hostPlatform.isLinux [ + # For various sandboxing tests that needs a statically-linked shell, + # etc. + busybox-sandbox-shell + # For Overlay FS tests need `mount`, `umount`, and `unshare`. + # TODO use `unixtools` to be precise over which executables instead? + util-linux + ]; + + nativeBuildInputs = finalAttrs.passthru.baseNativeBuildInputs ++ [ + nix-cli + ]; + + buildInputs = [ + nix-store + nix-expr + ]; + + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../../.version + '' + # TEMP hack for Meson before make is gone, where + # `src/nix-functional-tests` is during the transition a symlink and + # not the actual directory directory. + + '' + cd $(readlink -e $PWD) + echo $PWD | grep tests/functional + ''; + + mesonCheckFlags = [ + "--print-errorlogs" + ]; + + preCheck = + # See https://github.com/NixOS/nix/issues/2523 + # Occurs often in tests since https://github.com/NixOS/nix/pull/9900 + lib.optionalString stdenv.hostPlatform.isDarwin '' + export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES + ''; + + doCheck = true; + + installPhase = '' + mkdir $out + ''; + + meta = { + platforms = lib.platforms.unix; + }; + +} // lib.optionalAttrs (test-daemon != null) { + NIX_DAEMON_PACKAGE = test-daemon; +}) diff --git a/tests/functional/plugins.sh b/tests/functional/plugins.sh index ab4876df9..fc2d1907c 100755 --- a/tests/functional/plugins.sh +++ b/tests/functional/plugins.sh @@ -2,10 +2,11 @@ source common.sh -if [[ $BUILD_SHARED_LIBS != 1 ]]; then - skipTest "Plugins are not supported" -fi +for ext in so dylib; do + plugin="$PWD/plugins/libplugintest.$ext" + [[ -f "$plugin" ]] && break +done -res=$(nix --option setting-set true --option plugin-files $PWD/plugins/libplugintest* eval --expr builtins.anotherNull) +res=$(nix --option setting-set true --option plugin-files "$plugin" eval --expr builtins.anotherNull) [ "$res"x = "nullx" ] diff --git a/tests/functional/plugins/meson.build b/tests/functional/plugins/meson.build new file mode 100644 index 000000000..3d6b2f0e1 --- /dev/null +++ b/tests/functional/plugins/meson.build @@ -0,0 +1,16 @@ +libplugintest = shared_module( + 'plugintest', + 'plugintest.cc', + cpp_args : [ + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + # '-include', 'config-fetchers.hh', + '-include', 'config-expr.hh', + ], + dependencies : [ + dependency('nix-expr'), + ], + build_by_default : false, +) diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index 1df232b68..388018118 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -264,6 +264,23 @@ badExitCode=0 nixVersion="$(nix eval --impure --raw --expr 'builtins.nixVersion' --extra-experimental-features nix-command)" +# TODO: write a repl interacter for testing. Papering over the differences between readline / editline and between platforms is a pain. + +# I couldn't get readline and editline to agree on the newline before the prompt, +# so let's just force it to be one empty line. +stripEmptyLinesBeforePrompt() { + # --null-data: treat input as NUL-terminated instead of newline-terminated + sed --null-data 's/\n\n*nix-repl>/\n\nnix-repl>/g' +} + +# We don't get a final prompt on darwin, so we strip this as well. +stripFinalPrompt() { + # Strip the final prompt and/or any trailing spaces + sed --null-data \ + -e 's/\(.*[^\n]\)\n\n*nix-repl>[ \n]*$/\1/' \ + -e 's/[ \n]*$/\n/' +} + runRepl () { # That is right, we are also filtering out the testdir _without underscores_. @@ -275,8 +292,13 @@ runRepl () { testDirNoUnderscores="${testDir//_/}" # TODO: pass arguments to nix repl; see lang.sh + _NIX_TEST_RAW_MARKDOWN=1 \ + _NIX_TEST_REPL_ECHO=1 \ nix repl 2>&1 \ | stripColors \ + | tr -d '\0' \ + | stripEmptyLinesBeforePrompt \ + | stripFinalPrompt \ | sed \ -e "s@$testDir@/path/to/tests/functional@g" \ -e "s@$testDirNoUnderscores@/path/to/tests/functional@g" \ @@ -291,7 +313,10 @@ for test in $(cd "$testDir/repl"; echo *.in); do in="$testDir/repl/$test.in" actual="$testDir/repl/$test.actual" expected="$testDir/repl/$test.expected" - (cd "$testDir/repl"; set +x; runRepl 2>&1) < "$in" > "$actual" + (cd "$testDir/repl"; set +x; runRepl 2>&1) < "$in" > "$actual" || { + echo "FAIL: $test (exit code $?)" >&2 + badExitCode=1 + } diffAndAcceptInner "$test" "$actual" "$expected" done diff --git a/tests/functional/repl/doc-comment-curried-args.expected b/tests/functional/repl/doc-comment-curried-args.expected index c10c171e1..56607e911 100644 --- a/tests/functional/repl/doc-comment-curried-args.expected +++ b/tests/functional/repl/doc-comment-curried-args.expected @@ -1,24 +1,28 @@ Nix Type :? for help. + +nix-repl> :l doc-comments.nix Added variables. -Function curriedArgs - … defined at - /path/to/tests/functional/repl/doc-comments.nix:48:5 +nix-repl> :doc curriedArgs +Function `curriedArgs`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:48:5 - A documented function. +A documented function. +nix-repl> x = curriedArgs 1 +nix-repl> "Note that users may not expect this to behave as it currently does" "Note that users may not expect this to behave as it currently does" -Function curriedArgs - … defined at - /path/to/tests/functional/repl/doc-comments.nix:50:5 +nix-repl> :doc x +Function `curriedArgs`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:50:5 - The function returned by applying once +The function returned by applying once -"This won't produce documentation, because we can't actually document arbitrary values" +nix-repl> "This won't produce docs; no support for arbitrary values" +"This won't produce docs; no support for arbitrary values" +nix-repl> :doc x 2 error: value does not have documentation - - diff --git a/tests/functional/repl/doc-comment-curried-args.in b/tests/functional/repl/doc-comment-curried-args.in index 8dbbfc370..06ba21dcc 100644 --- a/tests/functional/repl/doc-comment-curried-args.in +++ b/tests/functional/repl/doc-comment-curried-args.in @@ -3,5 +3,5 @@ x = curriedArgs 1 "Note that users may not expect this to behave as it currently does" :doc x -"This won't produce documentation, because we can't actually document arbitrary values" +"This won't produce docs; no support for arbitrary values" :doc x 2 diff --git a/tests/functional/repl/doc-comment-formals.expected b/tests/functional/repl/doc-comment-formals.expected index 704c0050b..1024919f4 100644 --- a/tests/functional/repl/doc-comment-formals.expected +++ b/tests/functional/repl/doc-comment-formals.expected @@ -1,13 +1,14 @@ Nix Type :? for help. + +nix-repl> :l doc-comments.nix Added variables. +nix-repl> "Note that this is not yet complete" "Note that this is not yet complete" -Function documentedFormals - … defined at - /path/to/tests/functional/repl/doc-comments.nix:57:5 - - Finds x - +nix-repl> :doc documentedFormals +Function `documentedFormals`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:57:5 +Finds x diff --git a/tests/functional/repl/doc-comment-function.expected b/tests/functional/repl/doc-comment-function.expected index 5ec465a96..3889c4f78 100644 --- a/tests/functional/repl/doc-comment-function.expected +++ b/tests/functional/repl/doc-comment-function.expected @@ -1,8 +1,7 @@ Nix Type :? for help. -Function defined at - /path/to/tests/functional/repl/doc-comment-function.nix:2:1 - - A doc comment for a file that only contains a function +nix-repl> :doc import ./doc-comment-function.nix +Function defined at /path/to/tests/functional/repl/doc-comment-function.nix:2:1 +A doc comment for a file that only contains a function diff --git a/tests/functional/repl/doc-compact.expected b/tests/functional/repl/doc-compact.expected index 4b05b653c..79f1fd44f 100644 --- a/tests/functional/repl/doc-compact.expected +++ b/tests/functional/repl/doc-compact.expected @@ -1,11 +1,11 @@ Nix Type :? for help. + +nix-repl> :l doc-comments.nix Added variables. -Function compact - … defined at - /path/to/tests/functional/repl/doc-comments.nix:18:20 - - boom - +nix-repl> :doc compact +Function `compact`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:18:20 +boom diff --git a/tests/functional/repl/doc-constant.expected b/tests/functional/repl/doc-constant.expected index c66558333..5787e04dc 100644 --- a/tests/functional/repl/doc-constant.expected +++ b/tests/functional/repl/doc-constant.expected @@ -1,23 +1,27 @@ Nix Type :? for help. + +nix-repl> :l doc-comments.nix Added variables. +nix-repl> :doc constant error: value does not have documentation -Attribute version +nix-repl> :doc lib.version +Attribute `version` - … defined at - /path/to/tests/functional/repl/doc-comments.nix:30:3 + … defined at /path/to/tests/functional/repl/doc-comments.nix:30:3 - Immovably fixed. +Immovably fixed. -Attribute empty +nix-repl> :doc lib.attr.empty +Attribute `empty` - … defined at - /path/to/tests/functional/repl/doc-comments.nix:33:3 + … defined at /path/to/tests/functional/repl/doc-comments.nix:33:3 - Unchangeably constant. +Unchangeably constant. +nix-repl> :doc lib.attr.undocument error: … while evaluating the attribute 'attr.undocument' at /path/to/tests/functional/repl/doc-comments.nix:33:3: @@ -32,59 +36,65 @@ error: | ^ Did you mean undocumented? -Attribute constant +nix-repl> :doc (import ./doc-comments.nix).constant +Attribute `constant` - … defined at - /path/to/tests/functional/repl/doc-comments.nix:27:3 + … defined at /path/to/tests/functional/repl/doc-comments.nix:27:3 - Firmly rigid. +Firmly rigid. -Attribute version +nix-repl> :doc (import ./doc-comments.nix).lib.version +Attribute `version` - … defined at - /path/to/tests/functional/repl/doc-comments.nix:30:3 + … defined at /path/to/tests/functional/repl/doc-comments.nix:30:3 - Immovably fixed. +Immovably fixed. -Attribute empty +nix-repl> :doc (import ./doc-comments.nix).lib.attr.empty +Attribute `empty` - … defined at - /path/to/tests/functional/repl/doc-comments.nix:33:3 + … defined at /path/to/tests/functional/repl/doc-comments.nix:33:3 - Unchangeably constant. +Unchangeably constant. -Attribute undocumented +nix-repl> :doc (import ./doc-comments.nix).lib.attr.undocumented +Attribute `undocumented` - … defined at - /path/to/tests/functional/repl/doc-comments.nix:35:3 + … defined at /path/to/tests/functional/repl/doc-comments.nix:35:3 - No documentation found. +No documentation found. +nix-repl> :doc missing error: undefined variable 'missing' at «string»:1:1: 1| missing | ^ +nix-repl> :doc constanz error: undefined variable 'constanz' at «string»:1:1: 1| constanz | ^ +nix-repl> :doc missing.attr error: undefined variable 'missing' at «string»:1:1: 1| missing.attr | ^ +nix-repl> :doc lib.missing error: attribute 'missing' missing at «string»:1:1: 1| lib.missing | ^ +nix-repl> :doc lib.missing.attr error: attribute 'missing' missing at «string»:1:1: 1| lib.missing.attr | ^ +nix-repl> :doc lib.attr.undocumental error: … while evaluating the attribute 'attr.undocumental' at /path/to/tests/functional/repl/doc-comments.nix:33:3: @@ -98,5 +108,3 @@ error: 1| lib.attr.undocumental | ^ Did you mean undocumented? - - diff --git a/tests/functional/repl/doc-floatedIn.expected b/tests/functional/repl/doc-floatedIn.expected index 30f135725..82bb80b95 100644 --- a/tests/functional/repl/doc-floatedIn.expected +++ b/tests/functional/repl/doc-floatedIn.expected @@ -1,11 +1,11 @@ Nix Type :? for help. + +nix-repl> :l doc-comments.nix Added variables. -Function floatedIn - … defined at - /path/to/tests/functional/repl/doc-comments.nix:16:5 - - This also works. - +nix-repl> :doc floatedIn +Function `floatedIn`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:16:5 +This also works. diff --git a/tests/functional/repl/doc-lambda-flavors.expected b/tests/functional/repl/doc-lambda-flavors.expected index 43f483ce9..ab5c95639 100644 --- a/tests/functional/repl/doc-lambda-flavors.expected +++ b/tests/functional/repl/doc-lambda-flavors.expected @@ -1,29 +1,29 @@ Nix Type :? for help. + +nix-repl> :l doc-comments.nix Added variables. -Function nonStrict - … defined at - /path/to/tests/functional/repl/doc-comments.nix:37:70 +nix-repl> :doc nonStrict +Function `nonStrict`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:37:70 - My syntax is not strict, but I'm strict anyway. +My syntax is not strict, but I'm strict anyway. -Function strict - … defined at - /path/to/tests/functional/repl/doc-comments.nix:38:63 +nix-repl> :doc strict +Function `strict`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:38:63 - I don't have to be strict, but I am anyway. +I don't have to be strict, but I am anyway. -Function strictPre - … defined at - /path/to/tests/functional/repl/doc-comments.nix:40:48 +nix-repl> :doc strictPre +Function `strictPre`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:40:48 - Here's one way to do this - -Function strictPost - … defined at - /path/to/tests/functional/repl/doc-comments.nix:41:53 - - Here's another way to do this +Here's one way to do this +nix-repl> :doc strictPost +Function `strictPost`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:41:53 +Here's another way to do this diff --git a/tests/functional/repl/doc-measurement.expected b/tests/functional/repl/doc-measurement.expected index 8598aaedb..555cac9a2 100644 --- a/tests/functional/repl/doc-measurement.expected +++ b/tests/functional/repl/doc-measurement.expected @@ -1,11 +1,11 @@ Nix Type :? for help. + +nix-repl> :l doc-comments.nix Added variables. -Function measurement - … defined at - /path/to/tests/functional/repl/doc-comments.nix:13:17 - - 👈 precisely this wide 👉 - +nix-repl> :doc measurement +Function `measurement`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:13:17 +👈 precisely this wide 👉 diff --git a/tests/functional/repl/doc-multiply.expected b/tests/functional/repl/doc-multiply.expected index db82af91f..21523e24c 100644 --- a/tests/functional/repl/doc-multiply.expected +++ b/tests/functional/repl/doc-multiply.expected @@ -1,15 +1,17 @@ Nix Type :? for help. + +nix-repl> :l doc-comments.nix Added variables. -Function multiply - … defined at - /path/to/tests/functional/repl/doc-comments.nix:10:14 - - Perform arithmetic multiplication. It's kind of like - repeated addition, very neat. - - | multiply 2 3 - | => 6 +nix-repl> :doc multiply +Function `multiply`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:10:14 +Perform *arithmetic* multiplication. It's kind of like repeated **addition**, very neat. + +```nix +multiply 2 3 +=> 6 +``` diff --git a/tests/functional/repl/doc-unambiguous.expected b/tests/functional/repl/doc-unambiguous.expected index 825aa1ee1..0db5505d7 100644 --- a/tests/functional/repl/doc-unambiguous.expected +++ b/tests/functional/repl/doc-unambiguous.expected @@ -1,11 +1,11 @@ Nix Type :? for help. + +nix-repl> :l doc-comments.nix Added variables. -Function unambiguous - … defined at - /path/to/tests/functional/repl/doc-comments.nix:24:5 - - Very close - +nix-repl> :doc unambiguous +Function `unambiguous`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:24:5 +Very close diff --git a/tests/functional/repl/pretty-print-idempotent.expected b/tests/functional/repl/pretty-print-idempotent.expected index f38b9b569..311855dae 100644 --- a/tests/functional/repl/pretty-print-idempotent.expected +++ b/tests/functional/repl/pretty-print-idempotent.expected @@ -1,29 +1,37 @@ Nix Type :? for help. + +nix-repl> :l pretty-print-idempotent.nix Added variables. +nix-repl> oneDeep { homepage = "https://example.com"; } +nix-repl> oneDeep { homepage = "https://example.com"; } +nix-repl> twoDeep { layerOne = { ... }; } +nix-repl> twoDeep { layerOne = { ... }; } +nix-repl> oneDeepList [ "https://example.com" ] +nix-repl> oneDeepList [ "https://example.com" ] +nix-repl> twoDeepList [ [ ... ] ] +nix-repl> twoDeepList [ [ ... ] ] - - diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh index 915d973b0..e5fe9c136 100755 --- a/tests/functional/restricted.sh +++ b/tests/functional/restricted.sh @@ -10,12 +10,12 @@ nix-instantiate --restrict-eval --eval -E '1 + 2' nix-instantiate --restrict-eval ./simple.nix -I src=. nix-instantiate --restrict-eval ./simple.nix -I src1=simple.nix -I src2=config.nix -I src3=./simple.builder.sh +# no default NIX_PATH +(unset NIX_PATH; ! nix-instantiate --restrict-eval --find-file .) + (! nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix') nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I src=../.. -(! nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel') -nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel' -I src=../../src - expectStderr 1 nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile ' | grepQuiet "forbidden in restricted mode" nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile ' -I src=. diff --git a/tests/functional/signing.sh b/tests/functional/signing.sh index 890d1446f..8ec093a48 100755 --- a/tests/functional/signing.sh +++ b/tests/functional/signing.sh @@ -5,108 +5,108 @@ source common.sh clearStoreIfPossible clearCache -nix-store --generate-binary-cache-key cache1.example.org $TEST_ROOT/sk1 $TEST_ROOT/pk1 -pk1=$(cat $TEST_ROOT/pk1) -nix-store --generate-binary-cache-key cache2.example.org $TEST_ROOT/sk2 $TEST_ROOT/pk2 -pk2=$(cat $TEST_ROOT/pk2) +nix-store --generate-binary-cache-key cache1.example.org "$TEST_ROOT"/sk1 "$TEST_ROOT"/pk1 +pk1=$(cat "$TEST_ROOT"/pk1) +nix-store --generate-binary-cache-key cache2.example.org "$TEST_ROOT"/sk2 "$TEST_ROOT"/pk2 +pk2=$(cat "$TEST_ROOT"/pk2) # Build a path. outPath=$(nix-build dependencies.nix --no-out-link --secret-key-files "$TEST_ROOT/sk1 $TEST_ROOT/sk2") # Verify that the path got signed. -info=$(nix path-info --json $outPath) -echo $info | jq -e '.[] | .ultimate == true' +info=$(nix path-info --json "$outPath") +echo "$info" | jq -e '.[] | .ultimate == true' TODO_NixOS # looks like an actual bug? Following line fails on NixOS: -echo $info | jq -e '.[] | .signatures.[] | select(startswith("cache1.example.org"))' -echo $info | jq -e '.[] | .signatures.[] | select(startswith("cache2.example.org"))' +echo "$info" | jq -e '.[] | .signatures.[] | select(startswith("cache1.example.org"))' +echo "$info" | jq -e '.[] | .signatures.[] | select(startswith("cache2.example.org"))' # Test "nix store verify". -nix store verify -r $outPath +nix store verify -r "$outPath" -expect 2 nix store verify -r $outPath --sigs-needed 1 +expect 2 nix store verify -r "$outPath" --sigs-needed 1 -nix store verify -r $outPath --sigs-needed 1 --trusted-public-keys $pk1 +nix store verify -r "$outPath" --sigs-needed 1 --trusted-public-keys "$pk1" -expect 2 nix store verify -r $outPath --sigs-needed 2 --trusted-public-keys $pk1 +expect 2 nix store verify -r "$outPath" --sigs-needed 2 --trusted-public-keys "$pk1" -nix store verify -r $outPath --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" +nix store verify -r "$outPath" --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" nix store verify --all --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" # Build something unsigned. outPath2=$(nix-build simple.nix --no-out-link) -nix store verify -r $outPath +nix store verify -r "$outPath" # Verify that the path did not get signed but does have the ultimate bit. -info=$(nix path-info --json $outPath2) -echo $info | jq -e '.[] | .ultimate == true' -echo $info | jq -e '.[] | .signatures == []' +info=$(nix path-info --json "$outPath2") +echo "$info" | jq -e '.[] | .ultimate == true' +echo "$info" | jq -e '.[] | .signatures == []' # Test "nix store verify". -nix store verify -r $outPath2 +nix store verify -r "$outPath2" -expect 2 nix store verify -r $outPath2 --sigs-needed 1 +expect 2 nix store verify -r "$outPath2" --sigs-needed 1 -expect 2 nix store verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1 +expect 2 nix store verify -r "$outPath2" --sigs-needed 1 --trusted-public-keys "$pk1" # Test "nix store sign". -nix store sign --key-file $TEST_ROOT/sk1 $outPath2 +nix store sign --key-file "$TEST_ROOT"/sk1 "$outPath2" -nix store verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1 +nix store verify -r "$outPath2" --sigs-needed 1 --trusted-public-keys "$pk1" # Build something content-addressed. outPathCA=$(IMPURE_VAR1=foo IMPURE_VAR2=bar nix-build ./fixed.nix -A good.0 --no-out-link) -nix path-info --json $outPathCA | jq -e '.[] | .ca | startswith("fixed:md5:")' +nix path-info --json "$outPathCA" | jq -e '.[] | .ca | startswith("fixed:md5:")' # Content-addressed paths don't need signatures, so they verify # regardless of --sigs-needed. -nix store verify $outPathCA -nix store verify $outPathCA --sigs-needed 1000 +nix store verify "$outPathCA" +nix store verify "$outPathCA" --sigs-needed 1000 # Check that signing a content-addressed path doesn't overflow validSigs -nix store sign --key-file $TEST_ROOT/sk1 $outPathCA -nix store verify -r $outPathCA --sigs-needed 1000 --trusted-public-keys $pk1 +nix store sign --key-file "$TEST_ROOT"/sk1 "$outPathCA" +nix store verify -r "$outPathCA" --sigs-needed 1000 --trusted-public-keys "$pk1" # Copy to a binary cache. -nix copy --to file://$cacheDir $outPath2 +nix copy --to file://"$cacheDir" "$outPath2" # Verify that signatures got copied. -info=$(nix path-info --store file://$cacheDir --json $outPath2) -echo $info | jq -e '.[] | .ultimate == false' -echo $info | jq -e '.[] | .signatures.[] | select(startswith("cache1.example.org"))' -echo $info | expect 4 jq -e '.[] | .signatures.[] | select(startswith("cache2.example.org"))' +info=$(nix path-info --store file://"$cacheDir" --json "$outPath2") +echo "$info" | jq -e '.[] | .ultimate == false' +echo "$info" | jq -e '.[] | .signatures.[] | select(startswith("cache1.example.org"))' +echo "$info" | expect 4 jq -e '.[] | .signatures.[] | select(startswith("cache2.example.org"))' # Verify that adding a signature to a path in a binary cache works. -nix store sign --store file://$cacheDir --key-file $TEST_ROOT/sk2 $outPath2 -info=$(nix path-info --store file://$cacheDir --json $outPath2) -echo $info | jq -e '.[] | .signatures.[] | select(startswith("cache1.example.org"))' -echo $info | jq -e '.[] | .signatures.[] | select(startswith("cache2.example.org"))' +nix store sign --store file://"$cacheDir" --key-file "$TEST_ROOT"/sk2 "$outPath2" +info=$(nix path-info --store file://"$cacheDir" --json "$outPath2") +echo "$info" | jq -e '.[] | .signatures.[] | select(startswith("cache1.example.org"))' +echo "$info" | jq -e '.[] | .signatures.[] | select(startswith("cache2.example.org"))' # Copying to a diverted store should fail due to a lack of signatures by trusted keys. -chmod -R u+w $TEST_ROOT/store0 || true -rm -rf $TEST_ROOT/store0 +chmod -R u+w "$TEST_ROOT"/store0 || true +rm -rf "$TEST_ROOT"/store0 # Fails or very flaky only on GHA + macOS: # expectStderr 1 nix copy --to $TEST_ROOT/store0 $outPath | grepQuiet -E 'cannot add path .* because it lacks a signature by a trusted key' # but this works: -(! nix copy --to $TEST_ROOT/store0 $outPath) +(! nix copy --to "$TEST_ROOT"/store0 "$outPath") # But succeed if we supply the public keys. -nix copy --to $TEST_ROOT/store0 $outPath --trusted-public-keys $pk1 +nix copy --to "$TEST_ROOT"/store0 "$outPath" --trusted-public-keys "$pk1" -expect 2 nix store verify --store $TEST_ROOT/store0 -r $outPath +expect 2 nix store verify --store "$TEST_ROOT"/store0 -r "$outPath" -nix store verify --store $TEST_ROOT/store0 -r $outPath --trusted-public-keys $pk1 -nix store verify --store $TEST_ROOT/store0 -r $outPath --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" +nix store verify --store "$TEST_ROOT"/store0 -r "$outPath" --trusted-public-keys "$pk1" +nix store verify --store "$TEST_ROOT"/store0 -r "$outPath" --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" # It should also succeed if we disable signature checking. -(! nix copy --to $TEST_ROOT/store0 $outPath2) -nix copy --to $TEST_ROOT/store0?require-sigs=false $outPath2 +(! nix copy --to "$TEST_ROOT"/store0 "$outPath2") +nix copy --to "$TEST_ROOT"/store0?require-sigs=false "$outPath2" # But signatures should still get copied. -nix store verify --store $TEST_ROOT/store0 -r $outPath2 --trusted-public-keys $pk1 +nix store verify --store "$TEST_ROOT"/store0 -r "$outPath2" --trusted-public-keys "$pk1" # Content-addressed stuff can be copied without signatures. -nix copy --to $TEST_ROOT/store0 $outPathCA +nix copy --to "$TEST_ROOT"/store0 "$outPathCA" diff --git a/tests/functional/simple.builder.sh b/tests/functional/simple.builder.sh index 569e8ca88..97abf0676 100644 --- a/tests/functional/simple.builder.sh +++ b/tests/functional/simple.builder.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + echo "PATH=$PATH" # Verify that the PATH is empty. @@ -5,7 +7,6 @@ if mkdir foo 2> /dev/null; then exit 1; fi # Set a PATH (!!! impure). export PATH=$goodPath +mkdir "$out" -mkdir $out - -echo "Hello World!" > $out/hello \ No newline at end of file +echo "Hello World!" > "$out"/hello diff --git a/tests/functional/simple.sh b/tests/functional/simple.sh index 86acca0c2..8afa369c2 100755 --- a/tests/functional/simple.sh +++ b/tests/functional/simple.sh @@ -12,7 +12,7 @@ outPath=$(nix-store -rvv "$drvPath") echo "output path is $outPath" -(! [ -w $outPath ]) +[[ ! -w $outPath ]] text=$(cat "$outPath/hello") if test "$text" != "Hello World!"; then exit 1; fi @@ -21,16 +21,16 @@ TODO_NixOS # Directed delete: $outPath is not reachable from a root, so it should # be deleteable. -nix-store --delete $outPath -(! [ -e $outPath/hello ]) +nix-store --delete "$outPath" +[[ ! -e $outPath/hello ]] -outPath="$(NIX_REMOTE=local?store=/foo\&real=$TEST_ROOT/real-store nix-instantiate --readonly-mode hash-check.nix)" +outPath="$(NIX_REMOTE='local?store=/foo&real='"$TEST_ROOT"'/real-store' nix-instantiate --readonly-mode hash-check.nix)" if test "$outPath" != "/foo/lfy1s6ca46rm5r6w4gg9hc0axiakjcnm-dependencies.drv"; then echo "hashDerivationModulo appears broken, got $outPath" exit 1 fi -outPath="$(NIX_REMOTE=local?store=/foo\&real=$TEST_ROOT/real-store nix-instantiate --readonly-mode big-derivation-attr.nix)" +outPath="$(NIX_REMOTE='local?store=/foo&real='"$TEST_ROOT"'/real-store' nix-instantiate --readonly-mode big-derivation-attr.nix)" if test "$outPath" != "/foo/xxiwa5zlaajv6xdjynf9yym9g319d6mn-big-derivation-attr.drv"; then echo "big-derivation-attr.nix hash appears broken, got $outPath. Memory corruption in large drv attr?" exit 1 diff --git a/tests/functional/ssh-relay.sh b/tests/functional/ssh-relay.sh index 059c66434..71b8ae9ab 100755 --- a/tests/functional/ssh-relay.sh +++ b/tests/functional/ssh-relay.sh @@ -2,10 +2,10 @@ source common.sh -echo foo > $TEST_ROOT/hello.sh +echo foo > "$TEST_ROOT"/hello.sh ssh_localhost=ssh://localhost -remote_store=?remote-store=$ssh_localhost +remote_store="?remote-store=$ssh_localhost" store=$ssh_localhost @@ -13,6 +13,6 @@ store+=$remote_store store+=$remote_store store+=$remote_store -out=$(nix store add-path --store "$store" $TEST_ROOT/hello.sh) +out=$(nix store add-path --store "$store" "$TEST_ROOT"/hello.sh) -[ foo = $(< $out) ] +[ foo = "$(< "$out")" ] diff --git a/tests/functional/store-info.sh b/tests/functional/store-info.sh index f37889fbb..beecc2dd9 100755 --- a/tests/functional/store-info.sh +++ b/tests/functional/store-info.sh @@ -8,12 +8,12 @@ STORE_INFO_JSON=$(nix store info --json) echo "$STORE_INFO" | grep "Store URL: ${NIX_REMOTE}" if [[ -v NIX_DAEMON_PACKAGE ]] && isDaemonNewer "2.7.0pre20220126"; then - DAEMON_VERSION=$($NIX_DAEMON_PACKAGE/bin/nix daemon --version | cut -d' ' -f3) + DAEMON_VERSION=$("$NIX_DAEMON_PACKAGE"/bin/nix daemon --version | cut -d' ' -f3) echo "$STORE_INFO" | grep "Version: $DAEMON_VERSION" [[ "$(echo "$STORE_INFO_JSON" | jq -r ".version")" == "$DAEMON_VERSION" ]] fi -expect 127 NIX_REMOTE=unix:$PWD/store nix store info || \ +expect 127 NIX_REMOTE=unix:"$PWD"/store nix store info || \ fail "nix store info on a non-existent store should fail" TODO_NixOS diff --git a/tests/functional/structured-attrs.sh b/tests/functional/structured-attrs.sh index ec1282668..64d136e99 100755 --- a/tests/functional/structured-attrs.sh +++ b/tests/functional/structured-attrs.sh @@ -8,17 +8,19 @@ requireDaemonNewerThan "2.4pre20210712" clearStoreIfPossible -rm -f $TEST_ROOT/result +rm -f "$TEST_ROOT"/result -nix-build structured-attrs.nix -A all -o $TEST_ROOT/result +nix-build structured-attrs.nix -A all -o "$TEST_ROOT"/result -[[ $(cat $TEST_ROOT/result/foo) = bar ]] -[[ $(cat $TEST_ROOT/result-dev/foo) = foo ]] +[[ $(cat "$TEST_ROOT"/result/foo) = bar ]] +[[ $(cat "$TEST_ROOT"/result-dev/foo) = foo ]] export NIX_BUILD_SHELL=$SHELL +# shellcheck disable=SC2016 env NIX_PATH=nixpkgs=shell.nix nix-shell structured-attrs-shell.nix \ --run 'test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"' +# shellcheck disable=SC2016 nix develop -f structured-attrs-shell.nix -c bash -c 'test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"' TODO_NixOS # following line fails. @@ -26,6 +28,7 @@ TODO_NixOS # following line fails. # `nix develop` is a slightly special way of dealing with environment vars, it parses # these from a shell-file exported from a derivation. This is to test especially `outputs` # (which is an associative array in thsi case) being fine. +# shellcheck disable=SC2016 nix develop -f structured-attrs-shell.nix -c bash -c 'test -n "$out"' nix print-dev-env -f structured-attrs-shell.nix | grepQuiet 'NIX_ATTRS_JSON_FILE=' diff --git a/tests/functional/substitute-with-invalid-ca.sh b/tests/functional/substitute-with-invalid-ca.sh index d8af67237..33432e95d 100755 --- a/tests/functional/substitute-with-invalid-ca.sh +++ b/tests/functional/substitute-with-invalid-ca.sh @@ -11,16 +11,16 @@ getRemoteNarInfo () { echo "$cacheDir/$(getHash "$1").narinfo" } -cat < $TEST_HOME/good.txt +cat < "$TEST_HOME"/good.txt I’m a good path EOF -cat < $TEST_HOME/bad.txt +cat < "$TEST_HOME"/bad.txt I’m a bad path EOF -good=$(nix-store --add $TEST_HOME/good.txt) -bad=$(nix-store --add $TEST_HOME/bad.txt) +good=$(nix-store --add "$TEST_HOME"/good.txt) +bad=$(nix-store --add "$TEST_HOME"/bad.txt) nix copy --to "$BINARY_CACHE" "$good" nix copy --to "$BINARY_CACHE" "$bad" nix-collect-garbage >/dev/null 2>&1 diff --git a/tests/functional/suggestions.sh b/tests/functional/suggestions.sh index 8db6f7b97..fbca93da8 100755 --- a/tests/functional/suggestions.sh +++ b/tests/functional/suggestions.sh @@ -37,7 +37,7 @@ NIX_BUILD_STDERR_WITH_NO_CLOSE_SUGGESTION=$(! nix build .\#bar 2>&1 1>/dev/null) [[ ! "$NIX_BUILD_STDERR_WITH_NO_CLOSE_SUGGESTION" =~ "Did you mean" ]] || \ fail "The nix build stderr shouldn’t suggest anything if there’s nothing relevant to suggest" -NIX_EVAL_STDERR_WITH_SUGGESTIONS=$(! nix build --impure --expr '(builtins.getFlake (builtins.toPath ./.)).packages.'$system'.fob' 2>&1 1>/dev/null) +NIX_EVAL_STDERR_WITH_SUGGESTIONS=$(! nix build --impure --expr '(builtins.getFlake (builtins.toPath ./.)).packages.'"$system"'.fob' 2>&1 1>/dev/null) [[ "$NIX_EVAL_STDERR_WITH_SUGGESTIONS" =~ "Did you mean one of fo1, fo2, foo or fooo?" ]] || \ fail "The evaluator should suggest the three closest possiblities" diff --git a/tests/functional/tarball.sh b/tests/functional/tarball.sh index ab357ac78..dee0a98f1 100755 --- a/tests/functional/tarball.sh +++ b/tests/functional/tarball.sh @@ -4,51 +4,51 @@ source common.sh clearStoreIfPossible -rm -rf $TEST_HOME +rm -rf "$TEST_HOME" tarroot=$TEST_ROOT/tarball -rm -rf $tarroot -mkdir -p $tarroot -cp dependencies.nix $tarroot/default.nix -cp config.nix dependencies.builder*.sh $tarroot/ -touch -d '@1000000000' $tarroot $tarroot/* +rm -rf "$tarroot" +mkdir -p "$tarroot" +cp dependencies.nix "$tarroot/default.nix" +cp config.nix dependencies.builder*.sh "$tarroot/" +touch -d '@1000000000' "$tarroot" "$tarroot"/* -hash=$(nix hash path $tarroot) +hash=$(nix hash path "$tarroot") test_tarball() { local ext="$1" local compressor="$2" tarball=$TEST_ROOT/tarball.tar$ext - (cd $TEST_ROOT && GNUTAR_REPRODUCIBLE= tar --mtime=$tarroot/default.nix --owner=0 --group=0 --numeric-owner --sort=name -c -f - tarball) | $compressor > $tarball + (cd "$TEST_ROOT" && GNUTAR_REPRODUCIBLE=1 tar --mtime="$tarroot"/default.nix --owner=0 --group=0 --numeric-owner --sort=name -c -f - tarball) | $compressor > "$tarball" - nix-env -f file://$tarball -qa --out-path | grepQuiet dependencies + nix-env -f file://"$tarball" -qa --out-path | grepQuiet dependencies - nix-build -o $TEST_ROOT/result file://$tarball + nix-build -o "$TEST_ROOT"/result file://"$tarball" - nix-build -o $TEST_ROOT/result '' -I foo=file://$tarball + nix-build -o "$TEST_ROOT"/result '' -I foo=file://"$tarball" - nix-build -o $TEST_ROOT/result -E "import (fetchTarball file://$tarball)" + nix-build -o "$TEST_ROOT"/result -E "import (fetchTarball file://$tarball)" # Do not re-fetch paths already present - nix-build -o $TEST_ROOT/result -E "import (fetchTarball { url = file:///does-not-exist/must-remain-unused/$tarball; sha256 = \"$hash\"; })" + nix-build -o "$TEST_ROOT"/result -E "import (fetchTarball { url = file:///does-not-exist/must-remain-unused/$tarball; sha256 = \"$hash\"; })" - nix-build -o $TEST_ROOT/result -E "import (fetchTree file://$tarball)" - nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })" - nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })" + nix-build -o "$TEST_ROOT"/result -E "import (fetchTree file://$tarball)" + nix-build -o "$TEST_ROOT"/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })" + nix-build -o "$TEST_ROOT"/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })" # Do not re-fetch paths already present - nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file:///does-not-exist/must-remain-unused/$tarball; narHash = \"$hash\"; })" - expectStderr 102 nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" | grep 'NAR hash mismatch in input' + nix-build -o "$TEST_ROOT"/result -E "import (fetchTree { type = \"tarball\"; url = file:///does-not-exist/must-remain-unused/$tarball; narHash = \"$hash\"; })" + expectStderr 102 nix-build -o "$TEST_ROOT"/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" | grep 'NAR hash mismatch in input' [[ $(nix eval --impure --expr "(fetchTree file://$tarball).lastModified") = 1000000000 ]] nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" >&2 nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" 2>&1 | grep 'true' - nix-instantiate --eval -E '1 + 2' -I fnord=file:///no-such-tarball.tar$ext - nix-instantiate --eval -E 'with ; 1 + 2' -I fnord=file:///no-such-tarball$ext - (! nix-instantiate --eval -E ' 1' -I fnord=file:///no-such-tarball$ext) + nix-instantiate --eval -E '1 + 2' -I fnord=file:///no-such-tarball.tar"$ext" + nix-instantiate --eval -E 'with ; 1 + 2' -I fnord=file:///no-such-tarball"$ext" + (! nix-instantiate --eval -E ' 1' -I fnord=file:///no-such-tarball"$ext") - nix-instantiate --eval -E '' -I fnord=file:///no-such-tarball$ext -I fnord=. + nix-instantiate --eval -E '' -I fnord=file:///no-such-tarball"$ext" -I fnord=. # Ensure that the `name` attribute isn’t accepted as that would mess # with the content-addressing @@ -83,3 +83,20 @@ path="$(nix flake prefetch --json "tarball+file://$(pwd)/tree.tar.gz" | jq -r .s [[ $(cat "$path/a/zzz") = bar ]] [[ $(cat "$path/c/aap") = bar ]] [[ $(cat "$path/fnord") = bar ]] + +# Test a tarball that has multiple top-level directories. +rm -rf "$TEST_ROOT/tar_root" +mkdir -p "$TEST_ROOT/tar_root" "$TEST_ROOT/tar_root/foo" "$TEST_ROOT/tar_root/bar" +tar cvf "$TEST_ROOT/tar.tar" -C "$TEST_ROOT/tar_root" . +path="$(nix flake prefetch --json "tarball+file://$TEST_ROOT/tar.tar" | jq -r .storePath)" +[[ -d "$path/foo" ]] +[[ -d "$path/bar" ]] + +# Test a tarball that has a single regular file. +rm -rf "$TEST_ROOT/tar_root" +mkdir -p "$TEST_ROOT/tar_root" +echo bar > "$TEST_ROOT/tar_root/foo" +chmod +x "$TEST_ROOT/tar_root/foo" +tar cvf "$TEST_ROOT/tar.tar" -C "$TEST_ROOT/tar_root" . +path="$(nix flake prefetch --refresh --json "tarball+file://$TEST_ROOT/tar.tar" | jq -r .storePath)" +[[ $(cat "$path/foo") = bar ]] diff --git a/tests/functional/test-infra.sh b/tests/functional/test-infra.sh index 37322b356..2da26b08c 100755 --- a/tests/functional/test-infra.sh +++ b/tests/functional/test-infra.sh @@ -13,6 +13,25 @@ expect 1 false # `expect` will fail when we get it wrong expect 1 expect 0 false +function ret() { + return "$1" +} + +# `expect` can call functions, not just executables +expect 0 ret 0 +expect 1 ret 1 + +# `expect` supports negative exit codes +expect -1 ret -1 + +# or high positive ones, equivalent to negative ones +expect 255 ret 255 +expect 255 ret -1 +expect -1 ret 255 + +# but it doesn't confuse negative exit codes with positive ones +expect 1 expect -10 ret 10 + noisyTrue () { echo YAY! >&2 true @@ -29,6 +48,7 @@ expectStderr 1 noisyFalse | grepQuiet NAY # `set -o pipefile` is enabled +# shellcheck disable=SC2317# shellcheck disable=SC2317 pipefailure () { # shellcheck disable=SC2216 true | false | true @@ -36,6 +56,7 @@ pipefailure () { expect 1 pipefailure unset pipefailure +# shellcheck disable=SC2317 pipefailure () { # shellcheck disable=SC2216 false | true | true @@ -63,12 +84,17 @@ expect 1 useUnbound # ! alone unfortunately negates `set -e`, but it works in functions: # shellcheck disable=SC2251 ! true +# shellcheck disable=SC2317 funBang () { ! true } expect 1 funBang unset funBang +# callerPrefix can be used by the test framework to improve error messages +# it reports about our call site here +echo "<[$(callerPrefix)]>" | grepQuiet -F "<[test-infra.sh:$LINENO: ]>" + # `grep -v -q` is not what we want for exit codes, but `grepInverse` is # Avoid `grep -v -q`. The following line proves the point, and if it fails, # we'll know that `grep` had a breaking change or `-v -q` may not be portable. @@ -85,3 +111,12 @@ unset res res=$(set -eu -o pipefail; echo foo | expect 1 grepQuietInverse foo | wc -c) (( res == 0 )) unset res + +# `grepQuiet` does not allow newlines in its arguments, because grep quietly +# treats them as multiple queries. +{ echo foo; echo bar; } | expectStderr -101 grepQuiet $'foo\nbar' \ + | grepQuiet -E 'test-infra\.sh:[0-9]+: in call to grepQuiet: newline not allowed in arguments; grep would try each line individually as if connected by an OR operator' + +# We took the blue pill and woke up in a world where `grep` is moderately safe. +expectStderr -101 grep $'foo\nbar' \ + | grepQuiet -E 'test-infra\.sh:[0-9]+: in call to grep: newline not allowed in arguments; grep would try each line individually as if connected by an OR operator' diff --git a/tests/functional/test-libstoreconsumer.sh b/tests/functional/test-libstoreconsumer.sh index d1a1accb6..2adead1c0 100755 --- a/tests/functional/test-libstoreconsumer.sh +++ b/tests/functional/test-libstoreconsumer.sh @@ -5,4 +5,4 @@ source common.sh drv="$(nix-instantiate simple.nix)" cat "$drv" out="$(./test-libstoreconsumer/test-libstoreconsumer "$drv")" -cat "$out/hello" | grep -F "Hello World!" +grep -F "Hello World!" < "$out/hello" diff --git a/tests/functional/test-libstoreconsumer/meson.build b/tests/functional/test-libstoreconsumer/meson.build new file mode 100644 index 000000000..7076127f7 --- /dev/null +++ b/tests/functional/test-libstoreconsumer/meson.build @@ -0,0 +1,14 @@ +libstoreconsumer_tester = executable( + 'test-libstoreconsumer', + 'main.cc', + cpp_args : [ + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + ], + dependencies : [ + dependency('nix-store'), + ], + build_by_default : false, +) diff --git a/tests/functional/timeout.sh b/tests/functional/timeout.sh index f42354538..ae47fdc96 100755 --- a/tests/functional/timeout.sh +++ b/tests/functional/timeout.sh @@ -9,7 +9,7 @@ needLocalStore "see #4813" messages=$(nix-build -Q timeout.nix -A infiniteLoop --timeout 2 2>&1) && status=0 || status=$? -if [ $status -ne 101 ]; then +if [ "$status" -ne 101 ]; then echo "error: 'nix-store' exited with '$status'; should have exited 101" # FIXME: https://github.com/NixOS/nix/issues/4813 diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index c0c7b42fd..66174c928 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -146,4 +146,6 @@ in functional_root = runNixOSTestFor "x86_64-linux" ./functional/as-root.nix; user-sandboxing = runNixOSTestFor "x86_64-linux" ./user-sandboxing; + + s3-binary-cache-store = runNixOSTestFor "x86_64-linux" ./s3-binary-cache-store.nix; } diff --git a/tests/nixos/nix-copy-closure.nix b/tests/nixos/nix-copy-closure.nix index 66cbfb033..b9daa0a1f 100644 --- a/tests/nixos/nix-copy-closure.nix +++ b/tests/nixos/nix-copy-closure.nix @@ -1,6 +1,6 @@ # Test ‘nix-copy-closure’. -{ lib, config, nixpkgs, hostPkgs, ... }: +{ lib, config, nixpkgs, ... }: let pkgs = config.nodes.client.nixpkgs.pkgs; diff --git a/tests/nixos/quick-build.nix b/tests/nixos/quick-build.nix index 37169b4e2..57e3b9cdb 100644 --- a/tests/nixos/quick-build.nix +++ b/tests/nixos/quick-build.nix @@ -27,13 +27,13 @@ in }; config = { - passthru.quickBuild = + passthru.quickBuild = let withQuickBuild = extendModules { modules = [{ quickBuild = true; }]; }; in withQuickBuild.config.test; defaults = { pkgs, ... }: { config = lib.mkIf test.config.quickBuild { - nix.package = pkgs.nix_noTests; + nix.package = pkgs.nixComponents.nix-cli; system.forbiddenDependenciesRegexes = [ # This would indicate that the quickBuild feature is broken. @@ -44,4 +44,4 @@ in }; }; }; -} \ No newline at end of file +} diff --git a/tests/nixos/remote-builds.nix b/tests/nixos/remote-builds.nix index 1661203ec..8ddf6ad02 100644 --- a/tests/nixos/remote-builds.nix +++ b/tests/nixos/remote-builds.nix @@ -104,7 +104,15 @@ in builder.succeed("mkdir -p -m 700 /root/.ssh") builder.copy_from_host("key.pub", "/root/.ssh/authorized_keys") builder.wait_for_unit("sshd") - client.succeed(f"ssh -o StrictHostKeyChecking=no {builder.name} 'echo hello world'") + # Make sure the builder can handle our login correctly + builder.wait_for_unit("multi-user.target") + # Make sure there's no funny business on the client either + # (should not be necessary, but we have reason to be careful) + client.wait_for_unit("multi-user.target") + client.succeed(f""" + ssh -o StrictHostKeyChecking=no {builder.name} \ + 'echo hello world on $(hostname)' >&2 + """) # Perform a build and check that it was performed on the builder. out = client.succeed( diff --git a/tests/nixos/s3-binary-cache-store.nix b/tests/nixos/s3-binary-cache-store.nix new file mode 100644 index 000000000..015457968 --- /dev/null +++ b/tests/nixos/s3-binary-cache-store.nix @@ -0,0 +1,63 @@ +{ lib, config, nixpkgs, ... }: + +let + pkgs = config.nodes.client.nixpkgs.pkgs; + + pkgA = pkgs.cowsay; + + accessKey = "BKIKJAA5BMMU2RHO6IBB"; + secretKey = "V7f1CwQqAcwo80UEIJEjc5gVQUSSx5ohQ9GSrr12"; + env = "AWS_ACCESS_KEY_ID=${accessKey} AWS_SECRET_ACCESS_KEY=${secretKey}"; + + storeUrl = "s3://my-cache?endpoint=http://server:9000®ion=eu-west-1"; + +in { + name = "nix-copy-closure"; + + nodes = + { server = + { config, lib, pkgs, ... }: + { virtualisation.writableStore = true; + virtualisation.additionalPaths = [ pkgA ]; + environment.systemPackages = [ pkgs.minio-client ]; + nix.extraOptions = "experimental-features = nix-command"; + services.minio = { + enable = true; + region = "eu-west-1"; + rootCredentialsFile = pkgs.writeText "minio-credentials-full" '' + MINIO_ROOT_USER=${accessKey} + MINIO_ROOT_PASSWORD=${secretKey} + ''; + }; + networking.firewall.allowedTCPPorts = [ 9000 ]; + }; + + client = + { config, pkgs, ... }: + { virtualisation.writableStore = true; + nix.extraOptions = "experimental-features = nix-command"; + }; + }; + + testScript = { nodes }: '' + # fmt: off + start_all() + + # Create a binary cache. + server.wait_for_unit("minio") + + server.succeed("mc config host add minio http://localhost:9000 ${accessKey} ${secretKey} --api s3v4") + server.succeed("mc mb minio/my-cache") + + server.succeed("${env} nix copy --to '${storeUrl}' ${pkgA}") + + # Copy a package from the binary cache. + client.fail("nix path-info ${pkgA}") + + client.succeed("${env} nix store info --store '${storeUrl}' >&2") + + client.succeed("${env} nix copy --no-check-sigs --from '${storeUrl}' ${pkgA}") + + client.succeed("nix path-info ${pkgA}") + ''; +} diff --git a/tests/unit/libexpr-support/local.mk b/tests/unit/libexpr-support/local.mk deleted file mode 100644 index 0501de33c..000000000 --- a/tests/unit/libexpr-support/local.mk +++ /dev/null @@ -1,23 +0,0 @@ -libraries += libexpr-test-support - -libexpr-test-support_NAME = libnixexpr-test-support - -libexpr-test-support_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libexpr-test-support_INSTALL_DIR := $(checklibdir) -else - libexpr-test-support_INSTALL_DIR := -endif - -libexpr-test-support_SOURCES := \ - $(wildcard $(d)/tests/*.cc) \ - $(wildcard $(d)/tests/value/*.cc) - -libexpr-test-support_CXXFLAGS += $(libexpr-tests_EXTRA_INCLUDES) - -libexpr-test-support_LIBS = \ - libstore-test-support libutil-test-support \ - libexpr libstore libutil - -libexpr-test-support_LDFLAGS := $(THREAD_LDFLAGS) -lrapidcheck diff --git a/tests/unit/libexpr-support/package.nix b/tests/unit/libexpr-support/package.nix index 0c966c55a..f53aa842f 100644 --- a/tests/unit/libexpr-support/package.nix +++ b/tests/unit/libexpr-support/package.nix @@ -66,12 +66,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/tests/unit/libexpr-support/tests/libexpr.hh b/tests/unit/libexpr-support/tests/libexpr.hh index 54a4a0e9e..045607e87 100644 --- a/tests/unit/libexpr-support/tests/libexpr.hh +++ b/tests/unit/libexpr-support/tests/libexpr.hh @@ -85,7 +85,7 @@ namespace nix { if (arg.type() != nInt) { return false; } - return arg.integer() == v; + return arg.integer().value == v; } MATCHER_P(IsFloatEq, v, fmt("The float is equal to \"%1%\"", v)) { diff --git a/tests/unit/libexpr/local.mk b/tests/unit/libexpr/local.mk deleted file mode 100644 index 1617e2823..000000000 --- a/tests/unit/libexpr/local.mk +++ /dev/null @@ -1,45 +0,0 @@ -check: libexpr-tests_RUN - -programs += libexpr-tests - -libexpr-tests_NAME := libnixexpr-tests - -libexpr-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libexpr-tests.xml - -libexpr-tests_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libexpr-tests_INSTALL_DIR := $(checkbindir) -else - libexpr-tests_INSTALL_DIR := -endif - -libexpr-tests_SOURCES := \ - $(wildcard $(d)/*.cc) \ - $(wildcard $(d)/value/*.cc) \ - $(wildcard $(d)/flake/*.cc) - -libexpr-tests_EXTRA_INCLUDES = \ - -I tests/unit/libexpr-support \ - -I tests/unit/libstore-support \ - -I tests/unit/libutil-support \ - $(INCLUDE_libexpr) \ - $(INCLUDE_libexprc) \ - $(INCLUDE_libfetchers) \ - $(INCLUDE_libstore) \ - $(INCLUDE_libstorec) \ - $(INCLUDE_libutil) \ - $(INCLUDE_libutilc) - -libexpr-tests_CXXFLAGS += $(libexpr-tests_EXTRA_INCLUDES) - -libexpr-tests_LIBS = \ - libexpr-test-support libstore-test-support libutil-test-support \ - libexpr libexprc libfetchers libstore libstorec libutil libutilc - -libexpr-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock - -ifdef HOST_WINDOWS - # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space - libexpr-tests_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024))) -endif diff --git a/tests/unit/libexpr/main.cc b/tests/unit/libexpr/main.cc index cf7fcf5a3..e3412d9ef 100644 --- a/tests/unit/libexpr/main.cc +++ b/tests/unit/libexpr/main.cc @@ -34,6 +34,9 @@ int main (int argc, char **argv) { setEnv("_NIX_TEST_NO_SANDBOX", "1"); #endif + // For pipe operator tests in trivial.cc + experimentalFeatureSettings.set("experimental-features", "pipe-operators"); + ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } diff --git a/tests/unit/libexpr/package.nix b/tests/unit/libexpr/package.nix index 015e3fbc6..e70ed7836 100644 --- a/tests/unit/libexpr/package.nix +++ b/tests/unit/libexpr/package.nix @@ -71,12 +71,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; passthru = { diff --git a/tests/unit/libexpr/trivial.cc b/tests/unit/libexpr/trivial.cc index 61ea71a0f..e455a571b 100644 --- a/tests/unit/libexpr/trivial.cc +++ b/tests/unit/libexpr/trivial.cc @@ -182,6 +182,60 @@ namespace nix { ASSERT_THAT(v, IsIntEq(15)); } + TEST_F(TrivialExpressionTest, forwardPipe) { + auto v = eval("1 |> builtins.add 2 |> builtins.mul 3"); + ASSERT_THAT(v, IsIntEq(9)); + } + + TEST_F(TrivialExpressionTest, backwardPipe) { + auto v = eval("builtins.add 1 <| builtins.mul 2 <| 3"); + ASSERT_THAT(v, IsIntEq(7)); + } + + TEST_F(TrivialExpressionTest, forwardPipeEvaluationOrder) { + auto v = eval("1 |> null |> (x: 2)"); + ASSERT_THAT(v, IsIntEq(2)); + } + + TEST_F(TrivialExpressionTest, backwardPipeEvaluationOrder) { + auto v = eval("(x: 1) <| null <| 2"); + ASSERT_THAT(v, IsIntEq(1)); + } + + TEST_F(TrivialExpressionTest, differentPipeOperatorsDoNotAssociate) { + ASSERT_THROW(eval("(x: 1) <| 2 |> (x: 3)"), ParseError); + } + + TEST_F(TrivialExpressionTest, differentPipeOperatorsParensLeft) { + auto v = eval("((x: 1) <| 2) |> (x: 3)"); + ASSERT_THAT(v, IsIntEq(3)); + } + + TEST_F(TrivialExpressionTest, differentPipeOperatorsParensRight) { + auto v = eval("(x: 1) <| (2 |> (x: 3))"); + ASSERT_THAT(v, IsIntEq(1)); + } + + TEST_F(TrivialExpressionTest, forwardPipeLowestPrecedence) { + auto v = eval("false -> true |> (x: !x)"); + ASSERT_THAT(v, IsFalse()); + } + + TEST_F(TrivialExpressionTest, backwardPipeLowestPrecedence) { + auto v = eval("(x: !x) <| false -> true"); + ASSERT_THAT(v, IsFalse()); + } + + TEST_F(TrivialExpressionTest, forwardPipeStrongerThanElse) { + auto v = eval("if true then 1 else 2 |> 3"); + ASSERT_THAT(v, IsIntEq(1)); + } + + TEST_F(TrivialExpressionTest, backwardPipeStrongerThanElse) { + auto v = eval("if true then 1 else 2 <| 3"); + ASSERT_THAT(v, IsIntEq(1)); + } + TEST_F(TrivialExpressionTest, bindOr) { auto v = eval("{ or = 1; }"); ASSERT_THAT(v, IsAttrsOfSize(1)); diff --git a/tests/unit/libfetchers/git-utils.cc b/tests/unit/libfetchers/git-utils.cc index d3547ec6a..de5110cc3 100644 --- a/tests/unit/libfetchers/git-utils.cc +++ b/tests/unit/libfetchers/git-utils.cc @@ -77,7 +77,7 @@ TEST_F(GitUtilsTest, sink_basic) // sink->createHardlink("foo-1.1/links/foo-2", CanonPath("foo-1.1/hello")); - auto result = sink->sync(); + auto result = repo->dereferenceSingletonDirectory(sink->sync()); auto accessor = repo->getAccessor(result, false); auto entries = accessor->readDirectory(CanonPath::root); ASSERT_EQ(entries.size(), 5); @@ -103,7 +103,7 @@ TEST_F(GitUtilsTest, sink_hardlink) sink->createHardlink(CanonPath("foo-1.1/link"), CanonPath("hello")); FAIL() << "Expected an exception"; } catch (const nix::Error & e) { - ASSERT_THAT(e.msg(), testing::HasSubstr("invalid hard link target")); + ASSERT_THAT(e.msg(), testing::HasSubstr("cannot find hard link target")); ASSERT_THAT(e.msg(), testing::HasSubstr("/hello")); ASSERT_THAT(e.msg(), testing::HasSubstr("foo-1.1/link")); } diff --git a/tests/unit/libfetchers/local.mk b/tests/unit/libfetchers/local.mk deleted file mode 100644 index 30aa142a5..000000000 --- a/tests/unit/libfetchers/local.mk +++ /dev/null @@ -1,37 +0,0 @@ -check: libfetchers-tests_RUN - -programs += libfetchers-tests - -libfetchers-tests_NAME = libnixfetchers-tests - -libfetchers-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libfetchers-tests.xml - -libfetchers-tests_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libfetchers-tests_INSTALL_DIR := $(checkbindir) -else - libfetchers-tests_INSTALL_DIR := -endif - -libfetchers-tests_SOURCES := $(wildcard $(d)/*.cc) - -libfetchers-tests_EXTRA_INCLUDES = \ - -I tests/unit/libstore-support \ - -I tests/unit/libutil-support \ - $(INCLUDE_libfetchers) \ - $(INCLUDE_libstore) \ - $(INCLUDE_libutil) - -libfetchers-tests_CXXFLAGS += $(libfetchers-tests_EXTRA_INCLUDES) - -libfetchers-tests_LIBS = \ - libstore-test-support libutil-test-support \ - libfetchers libstore libutil - -libfetchers-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) $(LIBGIT2_LIBS) - -ifdef HOST_WINDOWS - # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space - libfetchers-tests_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024))) -endif diff --git a/tests/unit/libfetchers/package.nix b/tests/unit/libfetchers/package.nix index cf75f68e5..ad512f562 100644 --- a/tests/unit/libfetchers/package.nix +++ b/tests/unit/libfetchers/package.nix @@ -69,12 +69,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; passthru = { diff --git a/tests/unit/libflake/local.mk b/tests/unit/libflake/local.mk deleted file mode 100644 index 590bcf7c0..000000000 --- a/tests/unit/libflake/local.mk +++ /dev/null @@ -1,43 +0,0 @@ -check: libflake-tests_RUN - -programs += libflake-tests - -libflake-tests_NAME := libnixflake-tests - -libflake-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libflake-tests.xml - -libflake-tests_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libflake-tests_INSTALL_DIR := $(checkbindir) -else - libflake-tests_INSTALL_DIR := -endif - -libflake-tests_SOURCES := \ - $(wildcard $(d)/*.cc) \ - $(wildcard $(d)/value/*.cc) \ - $(wildcard $(d)/flake/*.cc) - -libflake-tests_EXTRA_INCLUDES = \ - -I tests/unit/libflake-support \ - -I tests/unit/libstore-support \ - -I tests/unit/libutil-support \ - $(INCLUDE_libflake) \ - $(INCLUDE_libexpr) \ - $(INCLUDE_libfetchers) \ - $(INCLUDE_libstore) \ - $(INCLUDE_libutil) \ - -libflake-tests_CXXFLAGS += $(libflake-tests_EXTRA_INCLUDES) - -libflake-tests_LIBS = \ - libexpr-test-support libstore-test-support libutil-test-support \ - libflake libexpr libfetchers libstore libutil - -libflake-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock - -ifdef HOST_WINDOWS - # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space - libflake-tests_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024))) -endif diff --git a/tests/unit/libflake/package.nix b/tests/unit/libflake/package.nix index d2c9fdb89..0d63d2ff7 100644 --- a/tests/unit/libflake/package.nix +++ b/tests/unit/libflake/package.nix @@ -69,12 +69,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; passthru = { diff --git a/tests/unit/libstore-support/local.mk b/tests/unit/libstore-support/local.mk deleted file mode 100644 index 56dedd825..000000000 --- a/tests/unit/libstore-support/local.mk +++ /dev/null @@ -1,21 +0,0 @@ -libraries += libstore-test-support - -libstore-test-support_NAME = libnixstore-test-support - -libstore-test-support_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libstore-test-support_INSTALL_DIR := $(checklibdir) -else - libstore-test-support_INSTALL_DIR := -endif - -libstore-test-support_SOURCES := $(wildcard $(d)/tests/*.cc) - -libstore-test-support_CXXFLAGS += $(libstore-tests_EXTRA_INCLUDES) - -libstore-test-support_LIBS = \ - libutil-test-support \ - libstore libutil - -libstore-test-support_LDFLAGS := $(THREAD_LDFLAGS) -lrapidcheck diff --git a/tests/unit/libstore-support/package.nix b/tests/unit/libstore-support/package.nix index cb15cdd5f..f512db3ee 100644 --- a/tests/unit/libstore-support/package.nix +++ b/tests/unit/libstore-support/package.nix @@ -66,12 +66,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/tests/unit/libstore/local.mk b/tests/unit/libstore/local.mk deleted file mode 100644 index 8d3d6b0af..000000000 --- a/tests/unit/libstore/local.mk +++ /dev/null @@ -1,38 +0,0 @@ -check: libstore-tests_RUN - -programs += libstore-tests - -libstore-tests_NAME = libnixstore-tests - -libstore-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libstore-tests.xml - -libstore-tests_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libstore-tests_INSTALL_DIR := $(checkbindir) -else - libstore-tests_INSTALL_DIR := -endif - -libstore-tests_SOURCES := $(wildcard $(d)/*.cc) - -libstore-tests_EXTRA_INCLUDES = \ - -I tests/unit/libstore-support \ - -I tests/unit/libutil-support \ - $(INCLUDE_libstore) \ - $(INCLUDE_libstorec) \ - $(INCLUDE_libutil) \ - $(INCLUDE_libutilc) - -libstore-tests_CXXFLAGS += $(libstore-tests_EXTRA_INCLUDES) - -libstore-tests_LIBS = \ - libstore-test-support libutil-test-support \ - libstore libstorec libutil libutilc - -libstore-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) - -ifdef HOST_WINDOWS - # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space - libstore-tests_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024))) -endif diff --git a/tests/unit/libstore/package.nix b/tests/unit/libstore/package.nix index 39bf77585..7560a5b79 100644 --- a/tests/unit/libstore/package.nix +++ b/tests/unit/libstore/package.nix @@ -73,12 +73,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; passthru = { diff --git a/tests/unit/libstore/worker-protocol.cc b/tests/unit/libstore/worker-protocol.cc index c15120010..bbea9ed75 100644 --- a/tests/unit/libstore/worker-protocol.cc +++ b/tests/unit/libstore/worker-protocol.cc @@ -658,15 +658,15 @@ TEST_F(WorkerProtoTest, handshake_log) FdSink out { toServer.writeSide.get() }; FdSource in0 { toClient.readSide.get() }; TeeSource in { in0, toClientLog }; - clientResult = WorkerProto::BasicClientConnection::handshake( - out, in, defaultVersion); + clientResult = std::get<0>(WorkerProto::BasicClientConnection::handshake( + out, in, defaultVersion, {})); }); { FdSink out { toClient.writeSide.get() }; FdSource in { toServer.readSide.get() }; WorkerProto::BasicServerConnection::handshake( - out, in, defaultVersion); + out, in, defaultVersion, {}); }; thread.join(); @@ -675,6 +675,33 @@ TEST_F(WorkerProtoTest, handshake_log) }); } +TEST_F(WorkerProtoTest, handshake_features) +{ + Pipe toClient, toServer; + toClient.create(); + toServer.create(); + + std::tuple> clientResult; + + auto clientThread = std::thread([&]() { + FdSink out { toServer.writeSide.get() }; + FdSource in { toClient.readSide.get() }; + clientResult = WorkerProto::BasicClientConnection::handshake( + out, in, 123, {"bar", "aap", "mies", "xyzzy"}); + }); + + FdSink out { toClient.writeSide.get() }; + FdSource in { toServer.readSide.get() }; + auto daemonResult = WorkerProto::BasicServerConnection::handshake( + out, in, 456, {"foo", "bar", "xyzzy"}); + + clientThread.join(); + + EXPECT_EQ(clientResult, daemonResult); + EXPECT_EQ(std::get<0>(clientResult), 123); + EXPECT_EQ(std::get<1>(clientResult), std::set({"bar", "xyzzy"})); +} + /// Has to be a `BufferedSink` for handshake. struct NullBufferedSink : BufferedSink { void writeUnbuffered(std::string_view data) override { } @@ -686,8 +713,8 @@ TEST_F(WorkerProtoTest, handshake_client_replay) NullBufferedSink nullSink; StringSource in { toClientLog }; - auto clientResult = WorkerProto::BasicClientConnection::handshake( - nullSink, in, defaultVersion); + auto clientResult = std::get<0>(WorkerProto::BasicClientConnection::handshake( + nullSink, in, defaultVersion, {})); EXPECT_EQ(clientResult, defaultVersion); }); @@ -705,13 +732,13 @@ TEST_F(WorkerProtoTest, handshake_client_truncated_replay_throws) if (len < 8) { EXPECT_THROW( WorkerProto::BasicClientConnection::handshake( - nullSink, in, defaultVersion), + nullSink, in, defaultVersion, {}), EndOfFile); } else { // Not sure why cannot keep on checking for `EndOfFile`. EXPECT_THROW( WorkerProto::BasicClientConnection::handshake( - nullSink, in, defaultVersion), + nullSink, in, defaultVersion, {}), Error); } } @@ -734,17 +761,17 @@ TEST_F(WorkerProtoTest, handshake_client_corrupted_throws) // magic bytes don't match EXPECT_THROW( WorkerProto::BasicClientConnection::handshake( - nullSink, in, defaultVersion), + nullSink, in, defaultVersion, {}), Error); } else if (idx < 8 || idx >= 12) { // Number out of bounds EXPECT_THROW( WorkerProto::BasicClientConnection::handshake( - nullSink, in, defaultVersion), + nullSink, in, defaultVersion, {}), SerialisationError); } else { - auto ver = WorkerProto::BasicClientConnection::handshake( - nullSink, in, defaultVersion); + auto ver = std::get<0>(WorkerProto::BasicClientConnection::handshake( + nullSink, in, defaultVersion, {})); // `std::min` of this and the other version saves us EXPECT_EQ(ver, defaultVersion); } diff --git a/tests/unit/libutil-support/local.mk b/tests/unit/libutil-support/local.mk deleted file mode 100644 index 5f7835c9f..000000000 --- a/tests/unit/libutil-support/local.mk +++ /dev/null @@ -1,19 +0,0 @@ -libraries += libutil-test-support - -libutil-test-support_NAME = libnixutil-test-support - -libutil-test-support_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libutil-test-support_INSTALL_DIR := $(checklibdir) -else - libutil-test-support_INSTALL_DIR := -endif - -libutil-test-support_SOURCES := $(wildcard $(d)/tests/*.cc) - -libutil-test-support_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES) - -libutil-test-support_LIBS = libutil - -libutil-test-support_LDFLAGS := $(THREAD_LDFLAGS) -lrapidcheck diff --git a/tests/unit/libutil-support/package.nix b/tests/unit/libutil-support/package.nix index fdecdec72..1665804cb 100644 --- a/tests/unit/libutil-support/package.nix +++ b/tests/unit/libutil-support/package.nix @@ -64,12 +64,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; meta = { diff --git a/tests/unit/libutil-support/tests/gtest-with-params.hh b/tests/unit/libutil-support/tests/gtest-with-params.hh new file mode 100644 index 000000000..323a083fe --- /dev/null +++ b/tests/unit/libutil-support/tests/gtest-with-params.hh @@ -0,0 +1,54 @@ +#pragma once +// SPDX-FileCopyrightText: 2014 Emil Eriksson +// +// SPDX-License-Identifier: BSD-2-Clause +// +// The lion's share of this code is copy pasted directly out of RapidCheck +// headers, so the copyright is set accordingly. +/** + * @file Implements the ability to run a RapidCheck test under gtest with changed + * test parameters such as the number of tests to run. This is useful for + * running very large numbers of the extremely cheap property tests. + */ + +#include +#include +#include + +namespace rc::detail { + +using MakeTestParams = TestParams (*)(); + +template +void checkGTestWith(Testable && testable, MakeTestParams makeTestParams) +{ + const auto testInfo = ::testing::UnitTest::GetInstance()->current_test_info(); + detail::TestMetadata metadata; + metadata.id = std::string(testInfo->test_case_name()) + "/" + std::string(testInfo->name()); + metadata.description = std::string(testInfo->name()); + + const auto result = checkTestable(std::forward(testable), metadata, makeTestParams()); + + if (result.template is()) { + const auto success = result.template get(); + if (!success.distribution.empty()) { + printResultMessage(result, std::cout); + std::cout << std::endl; + } + } else { + std::ostringstream ss; + printResultMessage(result, ss); + FAIL() << ss.str() << std::endl; + } +} +} + +#define RC_GTEST_PROP_WITH_PARAMS(TestCase, Name, MakeParams, ArgList) \ + void rapidCheck_propImpl_##TestCase##_##Name ArgList; \ + \ + TEST(TestCase, Name) \ + { \ + ::rc::detail::checkGTestWith(&rapidCheck_propImpl_##TestCase##_##Name, MakeParams); \ + } \ + \ + void rapidCheck_propImpl_##TestCase##_##Name ArgList diff --git a/tests/unit/libutil/checked-arithmetic.cc b/tests/unit/libutil/checked-arithmetic.cc new file mode 100644 index 000000000..75018660d --- /dev/null +++ b/tests/unit/libutil/checked-arithmetic.cc @@ -0,0 +1,158 @@ +#include +#include +#include +#include +#include +#include + +#include + +#include "tests/gtest-with-params.hh" + +namespace rc { +using namespace nix; + +template +struct Arbitrary> +{ + static Gen> arbitrary() + { + return gen::arbitrary(); + } +}; + +} + +namespace nix::checked { + +// Pointer to member function! Mildly gross. +template +using Oper = Checked::Result (Checked::*)(T const other) const; + +template +using ReferenceOper = T (*)(T a, T b); + +/** + * Checks that performing an operation that overflows into an inaccurate result + * has the desired behaviour. + * + * TBig is a type large enough to represent all results of TSmall operations. + */ +template +void checkType(TSmall a_, TSmall b, Oper oper, ReferenceOper reference) +{ + // Sufficient to fit all values + TBig referenceResult = reference(a_, b); + constexpr const TSmall minV = std::numeric_limits::min(); + constexpr const TSmall maxV = std::numeric_limits::max(); + + Checked a{a_}; + auto result = (a.*(oper))(b); + + // Just truncate it to get the in-range result + RC_ASSERT(result.valueWrapping() == static_cast(referenceResult)); + + if (referenceResult > maxV || referenceResult < minV) { + RC_ASSERT(result.overflowed()); + RC_ASSERT(!result.valueChecked().has_value()); + } else { + RC_ASSERT(!result.overflowed()); + RC_ASSERT(result.valueChecked().has_value()); + RC_ASSERT(*result.valueChecked() == referenceResult); + } +} + +/** + * Checks that performing an operation that overflows into an inaccurate result + * has the desired behaviour. + * + * TBig is a type large enough to represent all results of TSmall operations. + */ +template +void checkDivision(TSmall a_, TSmall b) +{ + // Sufficient to fit all values + constexpr const TSmall minV = std::numeric_limits::min(); + + Checked a{a_}; + auto result = a / b; + + if (std::is_signed() && a_ == minV && b == -1) { + // This is the only possible overflow condition + RC_ASSERT(result.valueWrapping() == minV); + RC_ASSERT(result.overflowed()); + } else if (b == 0) { + RC_ASSERT(result.divideByZero()); + RC_ASSERT_THROWS_AS(result.valueWrapping(), nix::checked::DivideByZero); + RC_ASSERT(result.valueChecked() == std::nullopt); + } else { + TBig referenceResult = a_ / b; + auto result_ = result.valueChecked(); + RC_ASSERT(result_.has_value()); + RC_ASSERT(*result_ == referenceResult); + RC_ASSERT(result.valueWrapping() == referenceResult); + } +} + +/** Creates parameters that perform a more adequate number of checks to validate + * extremely cheap tests such as arithmetic tests */ +static rc::detail::TestParams makeParams() +{ + auto const & conf = rc::detail::configuration(); + auto newParams = conf.testParams; + newParams.maxSuccess = 10000; + return newParams; +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, add_unsigned, makeParams, (uint16_t a, uint16_t b)) +{ + checkType(a, b, &Checked::operator+, [](int32_t a, int32_t b) { return a + b; }); +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, add_signed, makeParams, (int16_t a, int16_t b)) +{ + checkType(a, b, &Checked::operator+, [](int32_t a, int32_t b) { return a + b; }); +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, sub_unsigned, makeParams, (uint16_t a, uint16_t b)) +{ + checkType(a, b, &Checked::operator-, [](int32_t a, int32_t b) { return a - b; }); +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, sub_signed, makeParams, (int16_t a, int16_t b)) +{ + checkType(a, b, &Checked::operator-, [](int32_t a, int32_t b) { return a - b; }); +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, mul_unsigned, makeParams, (uint16_t a, uint16_t b)) +{ + checkType(a, b, &Checked::operator*, [](int64_t a, int64_t b) { return a * b; }); +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, mul_signed, makeParams, (int16_t a, int16_t b)) +{ + checkType(a, b, &Checked::operator*, [](int64_t a, int64_t b) { return a * b; }); +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, div_unsigned, makeParams, (uint16_t a, uint16_t b)) +{ + checkDivision(a, b); +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, div_signed, makeParams, (int16_t a, int16_t b)) +{ + checkDivision(a, b); +} + +// Make absolutely sure that we check the special cases if the proptest +// generator does not come up with them. This one is especially important +// because it has very specific pairs required for the edge cases unlike the +// others. +TEST(Checked, div_signed_special_cases) +{ + checkDivision(std::numeric_limits::min(), -1); + checkDivision(std::numeric_limits::min(), 0); + checkDivision(0, 0); +} + +} diff --git a/tests/unit/libutil/executable-path.cc b/tests/unit/libutil/executable-path.cc new file mode 100644 index 000000000..8d182357d --- /dev/null +++ b/tests/unit/libutil/executable-path.cc @@ -0,0 +1,64 @@ +#include + +#include "executable-path.hh" + +namespace nix { + +#ifdef WIN32 +# define PATH_VAR_SEP L";" +#else +# define PATH_VAR_SEP ":" +#endif + +#define PATH_ENV_ROUND_TRIP(NAME, STRING_LIT, CXX_LIT) \ + TEST(ExecutablePath, NAME) \ + { \ + OsString s = STRING_LIT; \ + auto v = ExecutablePath::parse(s); \ + EXPECT_EQ(v, (ExecutablePath CXX_LIT)); \ + auto s2 = v.render(); \ + EXPECT_EQ(s2, s); \ + } + +PATH_ENV_ROUND_TRIP(emptyRoundTrip, OS_STR(""), ({})) + +PATH_ENV_ROUND_TRIP( + oneElemRoundTrip, + OS_STR("/foo"), + ({ + OS_STR("/foo"), + })) + +PATH_ENV_ROUND_TRIP( + twoElemsRoundTrip, + OS_STR("/foo" PATH_VAR_SEP "/bar"), + ({ + OS_STR("/foo"), + OS_STR("/bar"), + })) + +PATH_ENV_ROUND_TRIP( + threeElemsRoundTrip, + OS_STR("/foo" PATH_VAR_SEP "." PATH_VAR_SEP "/bar"), + ({ + OS_STR("/foo"), + OS_STR("."), + OS_STR("/bar"), + })) + +TEST(ExecutablePath, elementyElemNormalize) +{ + auto v = ExecutablePath::parse(PATH_VAR_SEP PATH_VAR_SEP PATH_VAR_SEP); + EXPECT_EQ( + v, + (ExecutablePath{{ + OS_STR("."), + OS_STR("."), + OS_STR("."), + OS_STR("."), + }})); + auto s2 = v.render(); + EXPECT_EQ(s2, OS_STR("." PATH_VAR_SEP "." PATH_VAR_SEP "." PATH_VAR_SEP ".")); +} + +} diff --git a/tests/unit/libutil/file-system.cc b/tests/unit/libutil/file-system.cc new file mode 100644 index 000000000..cfddaae1c --- /dev/null +++ b/tests/unit/libutil/file-system.cc @@ -0,0 +1,258 @@ +#include "util.hh" +#include "types.hh" +#include "file-system.hh" +#include "processes.hh" +#include "terminal.hh" +#include "strings.hh" + +#include +#include +#include + +#include + +#ifdef _WIN32 +# define FS_SEP "\\" +# define FS_ROOT "C:" FS_SEP // Need a mounted one, C drive is likely +#else +# define FS_SEP "/" +# define FS_ROOT FS_SEP +#endif + +#ifndef PATH_MAX +# define PATH_MAX 4096 +#endif + +namespace nix { + +/* ----------- tests for file-system.hh -------------------------------------*/ + +/* ---------------------------------------------------------------------------- + * absPath + * --------------------------------------------------------------------------*/ + +TEST(absPath, doesntChangeRoot) +{ + auto p = absPath(FS_ROOT); + + ASSERT_EQ(p, FS_ROOT); +} + +TEST(absPath, turnsEmptyPathIntoCWD) +{ + char cwd[PATH_MAX + 1]; + auto p = absPath(""); + + ASSERT_EQ(p, getcwd((char *) &cwd, PATH_MAX)); +} + +TEST(absPath, usesOptionalBasePathWhenGiven) +{ + char _cwd[PATH_MAX + 1]; + char * cwd = getcwd((char *) &_cwd, PATH_MAX); + + auto p = absPath("", cwd); + + ASSERT_EQ(p, cwd); +} + +TEST(absPath, isIdempotent) +{ + char _cwd[PATH_MAX + 1]; + char * cwd = getcwd((char *) &_cwd, PATH_MAX); + auto p1 = absPath(cwd); + auto p2 = absPath(p1); + + ASSERT_EQ(p1, p2); +} + +TEST(absPath, pathIsCanonicalised) +{ + auto path = FS_ROOT "some/path/with/trailing/dot/."; + auto p1 = absPath(path); + auto p2 = absPath(p1); + + ASSERT_EQ(p1, FS_ROOT "some" FS_SEP "path" FS_SEP "with" FS_SEP "trailing" FS_SEP "dot"); + ASSERT_EQ(p1, p2); +} + +/* ---------------------------------------------------------------------------- + * canonPath + * --------------------------------------------------------------------------*/ + +TEST(canonPath, removesTrailingSlashes) +{ + auto path = FS_ROOT "this/is/a/path//"; + auto p = canonPath(path); + + ASSERT_EQ(p, FS_ROOT "this" FS_SEP "is" FS_SEP "a" FS_SEP "path"); +} + +TEST(canonPath, removesDots) +{ + auto path = FS_ROOT "this/./is/a/path/./"; + auto p = canonPath(path); + + ASSERT_EQ(p, FS_ROOT "this" FS_SEP "is" FS_SEP "a" FS_SEP "path"); +} + +TEST(canonPath, removesDots2) +{ + auto path = FS_ROOT "this/a/../is/a////path/foo/.."; + auto p = canonPath(path); + + ASSERT_EQ(p, FS_ROOT "this" FS_SEP "is" FS_SEP "a" FS_SEP "path"); +} + +TEST(canonPath, requiresAbsolutePath) +{ + ASSERT_ANY_THROW(canonPath(".")); + ASSERT_ANY_THROW(canonPath("..")); + ASSERT_ANY_THROW(canonPath("../")); + ASSERT_DEATH({ canonPath(""); }, "path != \"\""); +} + +/* ---------------------------------------------------------------------------- + * dirOf + * --------------------------------------------------------------------------*/ + +TEST(dirOf, returnsEmptyStringForRoot) +{ + auto p = dirOf("/"); + + ASSERT_EQ(p, "/"); +} + +TEST(dirOf, returnsFirstPathComponent) +{ + auto p1 = dirOf("/dir/"); + ASSERT_EQ(p1, "/dir"); + auto p2 = dirOf("/dir"); + ASSERT_EQ(p2, "/"); + auto p3 = dirOf("/dir/.."); + ASSERT_EQ(p3, "/dir"); + auto p4 = dirOf("/dir/../"); + ASSERT_EQ(p4, "/dir/.."); +} + +/* ---------------------------------------------------------------------------- + * baseNameOf + * --------------------------------------------------------------------------*/ + +TEST(baseNameOf, emptyPath) +{ + auto p1 = baseNameOf(""); + ASSERT_EQ(p1, ""); +} + +TEST(baseNameOf, pathOnRoot) +{ + auto p1 = baseNameOf("/dir"); + ASSERT_EQ(p1, "dir"); +} + +TEST(baseNameOf, relativePath) +{ + auto p1 = baseNameOf("dir/foo"); + ASSERT_EQ(p1, "foo"); +} + +TEST(baseNameOf, pathWithTrailingSlashRoot) +{ + auto p1 = baseNameOf("/"); + ASSERT_EQ(p1, ""); +} + +TEST(baseNameOf, trailingSlash) +{ + auto p1 = baseNameOf("/dir/"); + ASSERT_EQ(p1, "dir"); +} + +TEST(baseNameOf, trailingSlashes) +{ + auto p1 = baseNameOf("/dir//"); + ASSERT_EQ(p1, "dir"); +} + +TEST(baseNameOf, absoluteNothingSlashNothing) +{ + auto p1 = baseNameOf("//"); + ASSERT_EQ(p1, ""); +} + +/* ---------------------------------------------------------------------------- + * isInDir + * --------------------------------------------------------------------------*/ + +TEST(isInDir, trivialCase) +{ + auto p1 = isInDir("/foo/bar", "/foo"); + ASSERT_EQ(p1, true); +} + +TEST(isInDir, notInDir) +{ + auto p1 = isInDir("/zes/foo/bar", "/foo"); + ASSERT_EQ(p1, false); +} + +// XXX: hm, bug or feature? :) Looking at the implementation +// this might be problematic. +TEST(isInDir, emptyDir) +{ + auto p1 = isInDir("/zes/foo/bar", ""); + ASSERT_EQ(p1, true); +} + +/* ---------------------------------------------------------------------------- + * isDirOrInDir + * --------------------------------------------------------------------------*/ + +TEST(isDirOrInDir, trueForSameDirectory) +{ + ASSERT_EQ(isDirOrInDir("/nix", "/nix"), true); + ASSERT_EQ(isDirOrInDir("/", "/"), true); +} + +TEST(isDirOrInDir, trueForEmptyPaths) +{ + ASSERT_EQ(isDirOrInDir("", ""), true); +} + +TEST(isDirOrInDir, falseForDisjunctPaths) +{ + ASSERT_EQ(isDirOrInDir("/foo", "/bar"), false); +} + +TEST(isDirOrInDir, relativePaths) +{ + ASSERT_EQ(isDirOrInDir("/foo/..", "/foo"), true); +} + +// XXX: while it is possible to use "." or ".." in the +// first argument this doesn't seem to work in the second. +TEST(isDirOrInDir, DISABLED_shouldWork) +{ + ASSERT_EQ(isDirOrInDir("/foo/..", "/foo/."), true); +} + +/* ---------------------------------------------------------------------------- + * pathExists + * --------------------------------------------------------------------------*/ + +TEST(pathExists, rootExists) +{ + ASSERT_TRUE(pathExists(FS_ROOT)); +} + +TEST(pathExists, cwdExists) +{ + ASSERT_TRUE(pathExists(".")); +} + +TEST(pathExists, bogusPathDoesNotExist) +{ + ASSERT_FALSE(pathExists("/schnitzel/darmstadt/pommes")); +} +} diff --git a/tests/unit/libutil/local.mk b/tests/unit/libutil/local.mk deleted file mode 100644 index 404f35cf1..000000000 --- a/tests/unit/libutil/local.mk +++ /dev/null @@ -1,37 +0,0 @@ -check: libutil-tests_RUN - -programs += libutil-tests - -libutil-tests_NAME = libnixutil-tests - -libutil-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libutil-tests.xml - -libutil-tests_DIR := $(d) - -ifeq ($(INSTALL_UNIT_TESTS), yes) - libutil-tests_INSTALL_DIR := $(checkbindir) -else - libutil-tests_INSTALL_DIR := -endif - -libutil-tests_SOURCES := $(wildcard $(d)/*.cc) - -libutil-tests_EXTRA_INCLUDES = \ - -I tests/unit/libutil-support \ - $(INCLUDE_libutil) \ - $(INCLUDE_libutilc) - -libutil-tests_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES) - -libutil-tests_LIBS = libutil-test-support libutil libutilc - -libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) - -ifdef HOST_WINDOWS - # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space - libutil-tests_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024))) -endif - -check: $(d)/data/git/check-data.sh.test - -$(eval $(call run-test,$(d)/data/git/check-data.sh)) diff --git a/tests/unit/libutil/meson.build b/tests/unit/libutil/meson.build index 7f024e6f2..83cec13ec 100644 --- a/tests/unit/libutil/meson.build +++ b/tests/unit/libutil/meson.build @@ -52,6 +52,7 @@ sources = files( 'closure.cc', 'compression.cc', 'config.cc', + 'executable-path.cc', 'file-content-address.cc', 'git.cc', 'hash.cc', @@ -61,11 +62,14 @@ sources = files( 'lru-cache.cc', 'nix_api_util.cc', 'pool.cc', + 'processes.cc', 'references.cc', 'spawn.cc', + 'strings.cc', 'suggestions.cc', - 'tests.cc', + 'terminal.cc', 'url.cc', + 'util.cc', 'xml-writer.cc', ) diff --git a/tests/unit/libutil/package.nix b/tests/unit/libutil/package.nix index c7827e74f..2fce5bfa8 100644 --- a/tests/unit/libutil/package.nix +++ b/tests/unit/libutil/package.nix @@ -70,12 +70,8 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - enableParallelBuilding = true; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - strictDeps = true; - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; passthru = { diff --git a/tests/unit/libutil/processes.cc b/tests/unit/libutil/processes.cc new file mode 100644 index 000000000..9033595e8 --- /dev/null +++ b/tests/unit/libutil/processes.cc @@ -0,0 +1,17 @@ +#include "processes.hh" + +#include + +namespace nix { + +/* ---------------------------------------------------------------------------- + * statusOk + * --------------------------------------------------------------------------*/ + +TEST(statusOk, zeroIsOk) +{ + ASSERT_EQ(statusOk(0), true); + ASSERT_EQ(statusOk(1), false); +} + +} // namespace nix diff --git a/tests/unit/libutil/strings.cc b/tests/unit/libutil/strings.cc index 47a20770e..8ceb16767 100644 --- a/tests/unit/libutil/strings.cc +++ b/tests/unit/libutil/strings.cc @@ -1,4 +1,5 @@ #include +#include #include "strings.hh" @@ -80,4 +81,268 @@ TEST(concatStringsSep, buildSingleString) ASSERT_EQ(concatStringsSep(",", strings), "this"); } +/* ---------------------------------------------------------------------------- + * dropEmptyInitThenConcatStringsSep + * --------------------------------------------------------------------------*/ + +TEST(dropEmptyInitThenConcatStringsSep, empty) +{ + Strings strings; + + ASSERT_EQ(dropEmptyInitThenConcatStringsSep(",", strings), ""); +} + +TEST(dropEmptyInitThenConcatStringsSep, buildCommaSeparatedString) +{ + Strings strings; + strings.push_back("this"); + strings.push_back("is"); + strings.push_back("great"); + + ASSERT_EQ(dropEmptyInitThenConcatStringsSep(",", strings), "this,is,great"); +} + +TEST(dropEmptyInitThenConcatStringsSep, buildStringWithEmptySeparator) +{ + Strings strings; + strings.push_back("this"); + strings.push_back("is"); + strings.push_back("great"); + + ASSERT_EQ(dropEmptyInitThenConcatStringsSep("", strings), "thisisgreat"); +} + +TEST(dropEmptyInitThenConcatStringsSep, buildSingleString) +{ + Strings strings; + strings.push_back("this"); + strings.push_back(""); + + ASSERT_EQ(dropEmptyInitThenConcatStringsSep(",", strings), "this,"); +} + +TEST(dropEmptyInitThenConcatStringsSep, emptyStrings) +{ + Strings strings; + strings.push_back(""); + strings.push_back(""); + + ASSERT_EQ(dropEmptyInitThenConcatStringsSep(",", strings), ""); +} + +/* ---------------------------------------------------------------------------- + * tokenizeString + * --------------------------------------------------------------------------*/ + +TEST(tokenizeString, empty) +{ + Strings expected = {}; + + ASSERT_EQ(tokenizeString(""), expected); +} + +TEST(tokenizeString, oneSep) +{ + Strings expected = {}; + + ASSERT_EQ(tokenizeString(" "), expected); +} + +TEST(tokenizeString, twoSep) +{ + Strings expected = {}; + + ASSERT_EQ(tokenizeString(" \n"), expected); +} + +TEST(tokenizeString, tokenizeSpacesWithDefaults) +{ + auto s = "foo bar baz"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s), expected); +} + +TEST(tokenizeString, tokenizeTabsWithDefaults) +{ + auto s = "foo\tbar\tbaz"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s), expected); +} + +TEST(tokenizeString, tokenizeTabsSpacesWithDefaults) +{ + auto s = "foo\t bar\t baz"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s), expected); +} + +TEST(tokenizeString, tokenizeTabsSpacesNewlineWithDefaults) +{ + auto s = "foo\t\n bar\t\n baz"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s), expected); +} + +TEST(tokenizeString, tokenizeTabsSpacesNewlineRetWithDefaults) +{ + auto s = "foo\t\n\r bar\t\n\r baz"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s), expected); + + auto s2 = "foo \t\n\r bar \t\n\r baz"; + Strings expected2 = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s2), expected2); +} + +TEST(tokenizeString, tokenizeWithCustomSep) +{ + auto s = "foo\n,bar\n,baz\n"; + Strings expected = {"foo\n", "bar\n", "baz\n"}; + + ASSERT_EQ(tokenizeString(s, ","), expected); +} + +TEST(tokenizeString, tokenizeSepAtStart) +{ + auto s = ",foo,bar,baz"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s, ","), expected); +} + +TEST(tokenizeString, tokenizeSepAtEnd) +{ + auto s = "foo,bar,baz,"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s, ","), expected); +} + +TEST(tokenizeString, tokenizeSepEmpty) +{ + auto s = "foo,,baz"; + Strings expected = {"foo", "baz"}; + + ASSERT_EQ(tokenizeString(s, ","), expected); +} + +/* ---------------------------------------------------------------------------- + * splitString + * --------------------------------------------------------------------------*/ + +TEST(splitString, empty) +{ + Strings expected = {""}; + + ASSERT_EQ(splitString("", " \t\n\r"), expected); +} + +TEST(splitString, oneSep) +{ + Strings expected = {"", ""}; + + ASSERT_EQ(splitString(" ", " \t\n\r"), expected); +} + +TEST(splitString, twoSep) +{ + Strings expected = {"", "", ""}; + + ASSERT_EQ(splitString(" \n", " \t\n\r"), expected); +} + +TEST(splitString, tokenizeSpacesWithSpaces) +{ + auto s = "foo bar baz"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(splitString(s, " \t\n\r"), expected); +} + +TEST(splitString, tokenizeTabsWithDefaults) +{ + auto s = "foo\tbar\tbaz"; + // Using it like this is weird, but shows the difference with tokenizeString, which also has this test + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(splitString(s, " \t\n\r"), expected); +} + +TEST(splitString, tokenizeTabsSpacesWithDefaults) +{ + auto s = "foo\t bar\t baz"; + // Using it like this is weird, but shows the difference with tokenizeString, which also has this test + Strings expected = {"foo", "", "bar", "", "baz"}; + + ASSERT_EQ(splitString(s, " \t\n\r"), expected); +} + +TEST(splitString, tokenizeTabsSpacesNewlineWithDefaults) +{ + auto s = "foo\t\n bar\t\n baz"; + // Using it like this is weird, but shows the difference with tokenizeString, which also has this test + Strings expected = {"foo", "", "", "bar", "", "", "baz"}; + + ASSERT_EQ(splitString(s, " \t\n\r"), expected); +} + +TEST(splitString, tokenizeTabsSpacesNewlineRetWithDefaults) +{ + auto s = "foo\t\n\r bar\t\n\r baz"; + // Using it like this is weird, but shows the difference with tokenizeString, which also has this test + Strings expected = {"foo", "", "", "", "bar", "", "", "", "baz"}; + + ASSERT_EQ(splitString(s, " \t\n\r"), expected); + + auto s2 = "foo \t\n\r bar \t\n\r baz"; + Strings expected2 = {"foo", "", "", "", "", "bar", "", "", "", "", "baz"}; + + ASSERT_EQ(splitString(s2, " \t\n\r"), expected2); +} + +TEST(splitString, tokenizeWithCustomSep) +{ + auto s = "foo\n,bar\n,baz\n"; + Strings expected = {"foo\n", "bar\n", "baz\n"}; + + ASSERT_EQ(splitString(s, ","), expected); +} + +TEST(splitString, tokenizeSepAtStart) +{ + auto s = ",foo,bar,baz"; + Strings expected = {"", "foo", "bar", "baz"}; + + ASSERT_EQ(splitString(s, ","), expected); +} + +TEST(splitString, tokenizeSepAtEnd) +{ + auto s = "foo,bar,baz,"; + Strings expected = {"foo", "bar", "baz", ""}; + + ASSERT_EQ(splitString(s, ","), expected); +} + +TEST(splitString, tokenizeSepEmpty) +{ + auto s = "foo,,baz"; + Strings expected = {"foo", "", "baz"}; + + ASSERT_EQ(splitString(s, ","), expected); +} + +// concatStringsSep sep . splitString sep = id if sep is 1 char +RC_GTEST_PROP(splitString, recoveredByConcatStringsSep, (const std::string & s)) +{ + RC_ASSERT(concatStringsSep("/", splitString(s, "/")) == s); + RC_ASSERT(concatStringsSep("a", splitString(s, "a")) == s); +} + } // namespace nix diff --git a/tests/unit/libutil/terminal.cc b/tests/unit/libutil/terminal.cc new file mode 100644 index 000000000..cdeb9fd94 --- /dev/null +++ b/tests/unit/libutil/terminal.cc @@ -0,0 +1,60 @@ +#include "util.hh" +#include "types.hh" +#include "terminal.hh" +#include "strings.hh" + +#include +#include + +#include + +namespace nix { + +/* ---------------------------------------------------------------------------- + * filterANSIEscapes + * --------------------------------------------------------------------------*/ + +TEST(filterANSIEscapes, emptyString) +{ + auto s = ""; + auto expected = ""; + + ASSERT_EQ(filterANSIEscapes(s), expected); +} + +TEST(filterANSIEscapes, doesntChangePrintableChars) +{ + auto s = "09 2q304ruyhr slk2-19024 kjsadh sar f"; + + ASSERT_EQ(filterANSIEscapes(s), s); +} + +TEST(filterANSIEscapes, filtersColorCodes) +{ + auto s = "\u001b[30m A \u001b[31m B \u001b[32m C \u001b[33m D \u001b[0m"; + + ASSERT_EQ(filterANSIEscapes(s, true, 2), " A"); + ASSERT_EQ(filterANSIEscapes(s, true, 3), " A "); + ASSERT_EQ(filterANSIEscapes(s, true, 4), " A "); + ASSERT_EQ(filterANSIEscapes(s, true, 5), " A B"); + ASSERT_EQ(filterANSIEscapes(s, true, 8), " A B C"); +} + +TEST(filterANSIEscapes, expandsTabs) +{ + auto s = "foo\tbar\tbaz"; + + ASSERT_EQ(filterANSIEscapes(s, true), "foo bar baz"); +} + +TEST(filterANSIEscapes, utf8) +{ + ASSERT_EQ(filterANSIEscapes("foobar", true, 5), "fooba"); + ASSERT_EQ(filterANSIEscapes("fóóbär", true, 6), "fóóbär"); + ASSERT_EQ(filterANSIEscapes("fóóbär", true, 5), "fóóbä"); + ASSERT_EQ(filterANSIEscapes("fóóbär", true, 3), "fóó"); + ASSERT_EQ(filterANSIEscapes("f€€bär", true, 4), "f€€b"); + ASSERT_EQ(filterANSIEscapes("f𐍈𐍈bär", true, 4), "f𐍈𐍈b"); +} + +} // namespace nix diff --git a/tests/unit/libutil/tests.cc b/tests/unit/libutil/tests.cc deleted file mode 100644 index 8a3ca8561..000000000 --- a/tests/unit/libutil/tests.cc +++ /dev/null @@ -1,699 +0,0 @@ -#include "util.hh" -#include "types.hh" -#include "file-system.hh" -#include "processes.hh" -#include "terminal.hh" - -#include -#include - -#include - -#ifdef _WIN32 -# define FS_SEP "\\" -# define FS_ROOT "C:" FS_SEP // Need a mounted one, C drive is likely -#else -# define FS_SEP "/" -# define FS_ROOT FS_SEP -#endif - -namespace nix { - -/* ----------- tests for util.hh ------------------------------------------------*/ - - /* ---------------------------------------------------------------------------- - * absPath - * --------------------------------------------------------------------------*/ - - TEST(absPath, doesntChangeRoot) { - auto p = absPath(FS_ROOT); - - ASSERT_EQ(p, FS_ROOT); - } - - - - - TEST(absPath, turnsEmptyPathIntoCWD) { - char cwd[PATH_MAX+1]; - auto p = absPath(""); - - ASSERT_EQ(p, getcwd((char*)&cwd, PATH_MAX)); - } - - TEST(absPath, usesOptionalBasePathWhenGiven) { - char _cwd[PATH_MAX+1]; - char* cwd = getcwd((char*)&_cwd, PATH_MAX); - - auto p = absPath("", cwd); - - ASSERT_EQ(p, cwd); - } - - TEST(absPath, isIdempotent) { - char _cwd[PATH_MAX+1]; - char* cwd = getcwd((char*)&_cwd, PATH_MAX); - auto p1 = absPath(cwd); - auto p2 = absPath(p1); - - ASSERT_EQ(p1, p2); - } - - - TEST(absPath, pathIsCanonicalised) { - auto path = FS_ROOT "some/path/with/trailing/dot/."; - auto p1 = absPath(path); - auto p2 = absPath(p1); - - ASSERT_EQ(p1, FS_ROOT "some" FS_SEP "path" FS_SEP "with" FS_SEP "trailing" FS_SEP "dot"); - ASSERT_EQ(p1, p2); - } - - /* ---------------------------------------------------------------------------- - * canonPath - * --------------------------------------------------------------------------*/ - - TEST(canonPath, removesTrailingSlashes) { - auto path = FS_ROOT "this/is/a/path//"; - auto p = canonPath(path); - - ASSERT_EQ(p, FS_ROOT "this" FS_SEP "is" FS_SEP "a" FS_SEP "path"); - } - - TEST(canonPath, removesDots) { - auto path = FS_ROOT "this/./is/a/path/./"; - auto p = canonPath(path); - - ASSERT_EQ(p, FS_ROOT "this" FS_SEP "is" FS_SEP "a" FS_SEP "path"); - } - - TEST(canonPath, removesDots2) { - auto path = FS_ROOT "this/a/../is/a////path/foo/.."; - auto p = canonPath(path); - - ASSERT_EQ(p, FS_ROOT "this" FS_SEP "is" FS_SEP "a" FS_SEP "path"); - } - - TEST(canonPath, requiresAbsolutePath) { - ASSERT_ANY_THROW(canonPath(".")); - ASSERT_ANY_THROW(canonPath("..")); - ASSERT_ANY_THROW(canonPath("../")); - ASSERT_DEATH({ canonPath(""); }, "path != \"\""); - } - - /* ---------------------------------------------------------------------------- - * dirOf - * --------------------------------------------------------------------------*/ - - TEST(dirOf, returnsEmptyStringForRoot) { - auto p = dirOf("/"); - - ASSERT_EQ(p, "/"); - } - - TEST(dirOf, returnsFirstPathComponent) { - auto p1 = dirOf("/dir/"); - ASSERT_EQ(p1, "/dir"); - auto p2 = dirOf("/dir"); - ASSERT_EQ(p2, "/"); - auto p3 = dirOf("/dir/.."); - ASSERT_EQ(p3, "/dir"); - auto p4 = dirOf("/dir/../"); - ASSERT_EQ(p4, "/dir/.."); - } - - /* ---------------------------------------------------------------------------- - * baseNameOf - * --------------------------------------------------------------------------*/ - - TEST(baseNameOf, emptyPath) { - auto p1 = baseNameOf(""); - ASSERT_EQ(p1, ""); - } - - TEST(baseNameOf, pathOnRoot) { - auto p1 = baseNameOf("/dir"); - ASSERT_EQ(p1, "dir"); - } - - TEST(baseNameOf, relativePath) { - auto p1 = baseNameOf("dir/foo"); - ASSERT_EQ(p1, "foo"); - } - - TEST(baseNameOf, pathWithTrailingSlashRoot) { - auto p1 = baseNameOf("/"); - ASSERT_EQ(p1, ""); - } - - TEST(baseNameOf, trailingSlash) { - auto p1 = baseNameOf("/dir/"); - ASSERT_EQ(p1, "dir"); - } - - TEST(baseNameOf, trailingSlashes) { - auto p1 = baseNameOf("/dir//"); - ASSERT_EQ(p1, "dir"); - } - - TEST(baseNameOf, absoluteNothingSlashNothing) { - auto p1 = baseNameOf("//"); - ASSERT_EQ(p1, ""); - } - - /* ---------------------------------------------------------------------------- - * isInDir - * --------------------------------------------------------------------------*/ - - TEST(isInDir, trivialCase) { - auto p1 = isInDir("/foo/bar", "/foo"); - ASSERT_EQ(p1, true); - } - - TEST(isInDir, notInDir) { - auto p1 = isInDir("/zes/foo/bar", "/foo"); - ASSERT_EQ(p1, false); - } - - // XXX: hm, bug or feature? :) Looking at the implementation - // this might be problematic. - TEST(isInDir, emptyDir) { - auto p1 = isInDir("/zes/foo/bar", ""); - ASSERT_EQ(p1, true); - } - - /* ---------------------------------------------------------------------------- - * isDirOrInDir - * --------------------------------------------------------------------------*/ - - TEST(isDirOrInDir, trueForSameDirectory) { - ASSERT_EQ(isDirOrInDir("/nix", "/nix"), true); - ASSERT_EQ(isDirOrInDir("/", "/"), true); - } - - TEST(isDirOrInDir, trueForEmptyPaths) { - ASSERT_EQ(isDirOrInDir("", ""), true); - } - - TEST(isDirOrInDir, falseForDisjunctPaths) { - ASSERT_EQ(isDirOrInDir("/foo", "/bar"), false); - } - - TEST(isDirOrInDir, relativePaths) { - ASSERT_EQ(isDirOrInDir("/foo/..", "/foo"), true); - } - - // XXX: while it is possible to use "." or ".." in the - // first argument this doesn't seem to work in the second. - TEST(isDirOrInDir, DISABLED_shouldWork) { - ASSERT_EQ(isDirOrInDir("/foo/..", "/foo/."), true); - - } - - /* ---------------------------------------------------------------------------- - * pathExists - * --------------------------------------------------------------------------*/ - - TEST(pathExists, rootExists) { - ASSERT_TRUE(pathExists(FS_ROOT)); - } - - TEST(pathExists, cwdExists) { - ASSERT_TRUE(pathExists(".")); - } - - TEST(pathExists, bogusPathDoesNotExist) { - ASSERT_FALSE(pathExists("/schnitzel/darmstadt/pommes")); - } - - /* ---------------------------------------------------------------------------- - * dropEmptyInitThenConcatStringsSep - * --------------------------------------------------------------------------*/ - - TEST(dropEmptyInitThenConcatStringsSep, buildCommaSeparatedString) { - Strings strings; - strings.push_back("this"); - strings.push_back("is"); - strings.push_back("great"); - - ASSERT_EQ(dropEmptyInitThenConcatStringsSep(",", strings), "this,is,great"); - } - - TEST(dropEmptyInitThenConcatStringsSep, buildStringWithEmptySeparator) { - Strings strings; - strings.push_back("this"); - strings.push_back("is"); - strings.push_back("great"); - - ASSERT_EQ(dropEmptyInitThenConcatStringsSep("", strings), "thisisgreat"); - } - - TEST(dropEmptyInitThenConcatStringsSep, buildSingleString) { - Strings strings; - strings.push_back("this"); - - ASSERT_EQ(dropEmptyInitThenConcatStringsSep(",", strings), "this"); - } - - /* ---------------------------------------------------------------------------- - * hasPrefix - * --------------------------------------------------------------------------*/ - - TEST(hasPrefix, emptyStringHasNoPrefix) { - ASSERT_FALSE(hasPrefix("", "foo")); - } - - TEST(hasPrefix, emptyStringIsAlwaysPrefix) { - ASSERT_TRUE(hasPrefix("foo", "")); - ASSERT_TRUE(hasPrefix("jshjkfhsadf", "")); - } - - TEST(hasPrefix, trivialCase) { - ASSERT_TRUE(hasPrefix("foobar", "foo")); - } - - /* ---------------------------------------------------------------------------- - * hasSuffix - * --------------------------------------------------------------------------*/ - - TEST(hasSuffix, emptyStringHasNoSuffix) { - ASSERT_FALSE(hasSuffix("", "foo")); - } - - TEST(hasSuffix, trivialCase) { - ASSERT_TRUE(hasSuffix("foo", "foo")); - ASSERT_TRUE(hasSuffix("foobar", "bar")); - } - - /* ---------------------------------------------------------------------------- - * base64Encode - * --------------------------------------------------------------------------*/ - - TEST(base64Encode, emptyString) { - ASSERT_EQ(base64Encode(""), ""); - } - - TEST(base64Encode, encodesAString) { - ASSERT_EQ(base64Encode("quod erat demonstrandum"), "cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="); - } - - TEST(base64Encode, encodeAndDecode) { - auto s = "quod erat demonstrandum"; - auto encoded = base64Encode(s); - auto decoded = base64Decode(encoded); - - ASSERT_EQ(decoded, s); - } - - TEST(base64Encode, encodeAndDecodeNonPrintable) { - char s[256]; - std::iota(std::rbegin(s), std::rend(s), 0); - - auto encoded = base64Encode(s); - auto decoded = base64Decode(encoded); - - EXPECT_EQ(decoded.length(), 255); - ASSERT_EQ(decoded, s); - } - - /* ---------------------------------------------------------------------------- - * base64Decode - * --------------------------------------------------------------------------*/ - - TEST(base64Decode, emptyString) { - ASSERT_EQ(base64Decode(""), ""); - } - - TEST(base64Decode, decodeAString) { - ASSERT_EQ(base64Decode("cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="), "quod erat demonstrandum"); - } - - TEST(base64Decode, decodeThrowsOnInvalidChar) { - ASSERT_THROW(base64Decode("cXVvZCBlcm_0IGRlbW9uc3RyYW5kdW0="), Error); - } - - /* ---------------------------------------------------------------------------- - * getLine - * --------------------------------------------------------------------------*/ - - TEST(getLine, all) { - { - auto [line, rest] = getLine("foo\nbar\nxyzzy"); - ASSERT_EQ(line, "foo"); - ASSERT_EQ(rest, "bar\nxyzzy"); - } - - { - auto [line, rest] = getLine("foo\r\nbar\r\nxyzzy"); - ASSERT_EQ(line, "foo"); - ASSERT_EQ(rest, "bar\r\nxyzzy"); - } - - { - auto [line, rest] = getLine("foo\n"); - ASSERT_EQ(line, "foo"); - ASSERT_EQ(rest, ""); - } - - { - auto [line, rest] = getLine("foo"); - ASSERT_EQ(line, "foo"); - ASSERT_EQ(rest, ""); - } - - { - auto [line, rest] = getLine(""); - ASSERT_EQ(line, ""); - ASSERT_EQ(rest, ""); - } - } - - /* ---------------------------------------------------------------------------- - * toLower - * --------------------------------------------------------------------------*/ - - TEST(toLower, emptyString) { - ASSERT_EQ(toLower(""), ""); - } - - TEST(toLower, nonLetters) { - auto s = "!@(*$#)(@#=\\234_"; - ASSERT_EQ(toLower(s), s); - } - - // std::tolower() doesn't handle unicode characters. In the context of - // store paths this isn't relevant but doesn't hurt to record this behavior - // here. - TEST(toLower, umlauts) { - auto s = "ÄÖÜ"; - ASSERT_EQ(toLower(s), "ÄÖÜ"); - } - - /* ---------------------------------------------------------------------------- - * string2Float - * --------------------------------------------------------------------------*/ - - TEST(string2Float, emptyString) { - ASSERT_EQ(string2Float(""), std::nullopt); - } - - TEST(string2Float, trivialConversions) { - ASSERT_EQ(string2Float("1.0"), 1.0); - - ASSERT_EQ(string2Float("0.0"), 0.0); - - ASSERT_EQ(string2Float("-100.25"), -100.25); - } - - /* ---------------------------------------------------------------------------- - * string2Int - * --------------------------------------------------------------------------*/ - - TEST(string2Int, emptyString) { - ASSERT_EQ(string2Int(""), std::nullopt); - } - - TEST(string2Int, trivialConversions) { - ASSERT_EQ(string2Int("1"), 1); - - ASSERT_EQ(string2Int("0"), 0); - - ASSERT_EQ(string2Int("-100"), -100); - } - - /* ---------------------------------------------------------------------------- - * renderSize - * --------------------------------------------------------------------------*/ - - TEST(renderSize, misc) { - ASSERT_EQ(renderSize(0, true), " 0.0 KiB"); - ASSERT_EQ(renderSize(100, true), " 0.1 KiB"); - ASSERT_EQ(renderSize(100), "0.1 KiB"); - ASSERT_EQ(renderSize(972, true), " 0.9 KiB"); - ASSERT_EQ(renderSize(973, true), " 1.0 KiB"); // FIXME: should round down - ASSERT_EQ(renderSize(1024, true), " 1.0 KiB"); - ASSERT_EQ(renderSize(1024 * 1024, true), "1024.0 KiB"); - ASSERT_EQ(renderSize(1100 * 1024, true), " 1.1 MiB"); - ASSERT_EQ(renderSize(2ULL * 1024 * 1024 * 1024, true), " 2.0 GiB"); - ASSERT_EQ(renderSize(2100ULL * 1024 * 1024 * 1024, true), " 2.1 TiB"); - } - -#ifndef _WIN32 // TODO re-enable on Windows, once we can start processes - /* ---------------------------------------------------------------------------- - * statusOk - * --------------------------------------------------------------------------*/ - - TEST(statusOk, zeroIsOk) { - ASSERT_EQ(statusOk(0), true); - ASSERT_EQ(statusOk(1), false); - } -#endif - - - /* ---------------------------------------------------------------------------- - * rewriteStrings - * --------------------------------------------------------------------------*/ - - TEST(rewriteStrings, emptyString) { - StringMap rewrites; - rewrites["this"] = "that"; - - ASSERT_EQ(rewriteStrings("", rewrites), ""); - } - - TEST(rewriteStrings, emptyRewrites) { - StringMap rewrites; - - ASSERT_EQ(rewriteStrings("this and that", rewrites), "this and that"); - } - - TEST(rewriteStrings, successfulRewrite) { - StringMap rewrites; - rewrites["this"] = "that"; - - ASSERT_EQ(rewriteStrings("this and that", rewrites), "that and that"); - } - - TEST(rewriteStrings, doesntOccur) { - StringMap rewrites; - rewrites["foo"] = "bar"; - - ASSERT_EQ(rewriteStrings("this and that", rewrites), "this and that"); - } - - /* ---------------------------------------------------------------------------- - * replaceStrings - * --------------------------------------------------------------------------*/ - - TEST(replaceStrings, emptyString) { - ASSERT_EQ(replaceStrings("", "this", "that"), ""); - ASSERT_EQ(replaceStrings("this and that", "", ""), "this and that"); - } - - TEST(replaceStrings, successfulReplace) { - ASSERT_EQ(replaceStrings("this and that", "this", "that"), "that and that"); - } - - TEST(replaceStrings, doesntOccur) { - ASSERT_EQ(replaceStrings("this and that", "foo", "bar"), "this and that"); - } - - /* ---------------------------------------------------------------------------- - * trim - * --------------------------------------------------------------------------*/ - - TEST(trim, emptyString) { - ASSERT_EQ(trim(""), ""); - } - - TEST(trim, removesWhitespace) { - ASSERT_EQ(trim("foo"), "foo"); - ASSERT_EQ(trim(" foo "), "foo"); - ASSERT_EQ(trim(" foo bar baz"), "foo bar baz"); - ASSERT_EQ(trim(" \t foo bar baz\n"), "foo bar baz"); - } - - /* ---------------------------------------------------------------------------- - * chomp - * --------------------------------------------------------------------------*/ - - TEST(chomp, emptyString) { - ASSERT_EQ(chomp(""), ""); - } - - TEST(chomp, removesWhitespace) { - ASSERT_EQ(chomp("foo"), "foo"); - ASSERT_EQ(chomp("foo "), "foo"); - ASSERT_EQ(chomp(" foo "), " foo"); - ASSERT_EQ(chomp(" foo bar baz "), " foo bar baz"); - ASSERT_EQ(chomp("\t foo bar baz\n"), "\t foo bar baz"); - } - - /* ---------------------------------------------------------------------------- - * quoteStrings - * --------------------------------------------------------------------------*/ - - TEST(quoteStrings, empty) { - Strings s = { }; - Strings expected = { }; - - ASSERT_EQ(quoteStrings(s), expected); - } - - TEST(quoteStrings, emptyStrings) { - Strings s = { "", "", "" }; - Strings expected = { "''", "''", "''" }; - ASSERT_EQ(quoteStrings(s), expected); - - } - - TEST(quoteStrings, trivialQuote) { - Strings s = { "foo", "bar", "baz" }; - Strings expected = { "'foo'", "'bar'", "'baz'" }; - - ASSERT_EQ(quoteStrings(s), expected); - } - - TEST(quoteStrings, quotedStrings) { - Strings s = { "'foo'", "'bar'", "'baz'" }; - Strings expected = { "''foo''", "''bar''", "''baz''" }; - - ASSERT_EQ(quoteStrings(s), expected); - } - - /* ---------------------------------------------------------------------------- - * tokenizeString - * --------------------------------------------------------------------------*/ - - TEST(tokenizeString, empty) { - Strings expected = { }; - - ASSERT_EQ(tokenizeString(""), expected); - } - - TEST(tokenizeString, tokenizeSpacesWithDefaults) { - auto s = "foo bar baz"; - Strings expected = { "foo", "bar", "baz" }; - - ASSERT_EQ(tokenizeString(s), expected); - } - - TEST(tokenizeString, tokenizeTabsWithDefaults) { - auto s = "foo\tbar\tbaz"; - Strings expected = { "foo", "bar", "baz" }; - - ASSERT_EQ(tokenizeString(s), expected); - } - - TEST(tokenizeString, tokenizeTabsSpacesWithDefaults) { - auto s = "foo\t bar\t baz"; - Strings expected = { "foo", "bar", "baz" }; - - ASSERT_EQ(tokenizeString(s), expected); - } - - TEST(tokenizeString, tokenizeTabsSpacesNewlineWithDefaults) { - auto s = "foo\t\n bar\t\n baz"; - Strings expected = { "foo", "bar", "baz" }; - - ASSERT_EQ(tokenizeString(s), expected); - } - - TEST(tokenizeString, tokenizeTabsSpacesNewlineRetWithDefaults) { - auto s = "foo\t\n\r bar\t\n\r baz"; - Strings expected = { "foo", "bar", "baz" }; - - ASSERT_EQ(tokenizeString(s), expected); - - auto s2 = "foo \t\n\r bar \t\n\r baz"; - Strings expected2 = { "foo", "bar", "baz" }; - - ASSERT_EQ(tokenizeString(s2), expected2); - } - - TEST(tokenizeString, tokenizeWithCustomSep) { - auto s = "foo\n,bar\n,baz\n"; - Strings expected = { "foo\n", "bar\n", "baz\n" }; - - ASSERT_EQ(tokenizeString(s, ","), expected); - } - - /* ---------------------------------------------------------------------------- - * get - * --------------------------------------------------------------------------*/ - - TEST(get, emptyContainer) { - StringMap s = { }; - auto expected = nullptr; - - ASSERT_EQ(get(s, "one"), expected); - } - - TEST(get, getFromContainer) { - StringMap s; - s["one"] = "yi"; - s["two"] = "er"; - auto expected = "yi"; - - ASSERT_EQ(*get(s, "one"), expected); - } - - TEST(getOr, emptyContainer) { - StringMap s = { }; - auto expected = "yi"; - - ASSERT_EQ(getOr(s, "one", "yi"), expected); - } - - TEST(getOr, getFromContainer) { - StringMap s; - s["one"] = "yi"; - s["two"] = "er"; - auto expected = "yi"; - - ASSERT_EQ(getOr(s, "one", "nope"), expected); - } - - /* ---------------------------------------------------------------------------- - * filterANSIEscapes - * --------------------------------------------------------------------------*/ - - TEST(filterANSIEscapes, emptyString) { - auto s = ""; - auto expected = ""; - - ASSERT_EQ(filterANSIEscapes(s), expected); - } - - TEST(filterANSIEscapes, doesntChangePrintableChars) { - auto s = "09 2q304ruyhr slk2-19024 kjsadh sar f"; - - ASSERT_EQ(filterANSIEscapes(s), s); - } - - TEST(filterANSIEscapes, filtersColorCodes) { - auto s = "\u001b[30m A \u001b[31m B \u001b[32m C \u001b[33m D \u001b[0m"; - - ASSERT_EQ(filterANSIEscapes(s, true, 2), " A" ); - ASSERT_EQ(filterANSIEscapes(s, true, 3), " A " ); - ASSERT_EQ(filterANSIEscapes(s, true, 4), " A " ); - ASSERT_EQ(filterANSIEscapes(s, true, 5), " A B" ); - ASSERT_EQ(filterANSIEscapes(s, true, 8), " A B C" ); - } - - TEST(filterANSIEscapes, expandsTabs) { - auto s = "foo\tbar\tbaz"; - - ASSERT_EQ(filterANSIEscapes(s, true), "foo bar baz" ); - } - - TEST(filterANSIEscapes, utf8) { - ASSERT_EQ(filterANSIEscapes("foobar", true, 5), "fooba"); - ASSERT_EQ(filterANSIEscapes("fóóbär", true, 6), "fóóbär"); - ASSERT_EQ(filterANSIEscapes("fóóbär", true, 5), "fóóbä"); - ASSERT_EQ(filterANSIEscapes("fóóbär", true, 3), "fóó"); - ASSERT_EQ(filterANSIEscapes("f€€bär", true, 4), "f€€b"); - ASSERT_EQ(filterANSIEscapes("f𐍈𐍈bär", true, 4), "f𐍈𐍈b"); - } - -} diff --git a/tests/unit/libutil/util.cc b/tests/unit/libutil/util.cc new file mode 100644 index 000000000..a3f7c720a --- /dev/null +++ b/tests/unit/libutil/util.cc @@ -0,0 +1,385 @@ +#include "util.hh" +#include "types.hh" +#include "file-system.hh" +#include "terminal.hh" +#include "strings.hh" + +#include +#include + +#include + +namespace nix { + +/* ----------- tests for util.hh --------------------------------------------*/ + +/* ---------------------------------------------------------------------------- + * hasPrefix + * --------------------------------------------------------------------------*/ + +TEST(hasPrefix, emptyStringHasNoPrefix) +{ + ASSERT_FALSE(hasPrefix("", "foo")); +} + +TEST(hasPrefix, emptyStringIsAlwaysPrefix) +{ + ASSERT_TRUE(hasPrefix("foo", "")); + ASSERT_TRUE(hasPrefix("jshjkfhsadf", "")); +} + +TEST(hasPrefix, trivialCase) +{ + ASSERT_TRUE(hasPrefix("foobar", "foo")); +} + +/* ---------------------------------------------------------------------------- + * hasSuffix + * --------------------------------------------------------------------------*/ + +TEST(hasSuffix, emptyStringHasNoSuffix) +{ + ASSERT_FALSE(hasSuffix("", "foo")); +} + +TEST(hasSuffix, trivialCase) +{ + ASSERT_TRUE(hasSuffix("foo", "foo")); + ASSERT_TRUE(hasSuffix("foobar", "bar")); +} + +/* ---------------------------------------------------------------------------- + * base64Encode + * --------------------------------------------------------------------------*/ + +TEST(base64Encode, emptyString) +{ + ASSERT_EQ(base64Encode(""), ""); +} + +TEST(base64Encode, encodesAString) +{ + ASSERT_EQ(base64Encode("quod erat demonstrandum"), "cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="); +} + +TEST(base64Encode, encodeAndDecode) +{ + auto s = "quod erat demonstrandum"; + auto encoded = base64Encode(s); + auto decoded = base64Decode(encoded); + + ASSERT_EQ(decoded, s); +} + +TEST(base64Encode, encodeAndDecodeNonPrintable) +{ + char s[256]; + std::iota(std::rbegin(s), std::rend(s), 0); + + auto encoded = base64Encode(s); + auto decoded = base64Decode(encoded); + + EXPECT_EQ(decoded.length(), 255); + ASSERT_EQ(decoded, s); +} + +/* ---------------------------------------------------------------------------- + * base64Decode + * --------------------------------------------------------------------------*/ + +TEST(base64Decode, emptyString) +{ + ASSERT_EQ(base64Decode(""), ""); +} + +TEST(base64Decode, decodeAString) +{ + ASSERT_EQ(base64Decode("cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="), "quod erat demonstrandum"); +} + +TEST(base64Decode, decodeThrowsOnInvalidChar) +{ + ASSERT_THROW(base64Decode("cXVvZCBlcm_0IGRlbW9uc3RyYW5kdW0="), Error); +} + +/* ---------------------------------------------------------------------------- + * getLine + * --------------------------------------------------------------------------*/ + +TEST(getLine, all) +{ + { + auto [line, rest] = getLine("foo\nbar\nxyzzy"); + ASSERT_EQ(line, "foo"); + ASSERT_EQ(rest, "bar\nxyzzy"); + } + + { + auto [line, rest] = getLine("foo\r\nbar\r\nxyzzy"); + ASSERT_EQ(line, "foo"); + ASSERT_EQ(rest, "bar\r\nxyzzy"); + } + + { + auto [line, rest] = getLine("foo\n"); + ASSERT_EQ(line, "foo"); + ASSERT_EQ(rest, ""); + } + + { + auto [line, rest] = getLine("foo"); + ASSERT_EQ(line, "foo"); + ASSERT_EQ(rest, ""); + } + + { + auto [line, rest] = getLine(""); + ASSERT_EQ(line, ""); + ASSERT_EQ(rest, ""); + } +} + +/* ---------------------------------------------------------------------------- + * toLower + * --------------------------------------------------------------------------*/ + +TEST(toLower, emptyString) +{ + ASSERT_EQ(toLower(""), ""); +} + +TEST(toLower, nonLetters) +{ + auto s = "!@(*$#)(@#=\\234_"; + ASSERT_EQ(toLower(s), s); +} + +// std::tolower() doesn't handle unicode characters. In the context of +// store paths this isn't relevant but doesn't hurt to record this behavior +// here. +TEST(toLower, umlauts) +{ + auto s = "ÄÖÜ"; + ASSERT_EQ(toLower(s), "ÄÖÜ"); +} + +/* ---------------------------------------------------------------------------- + * string2Float + * --------------------------------------------------------------------------*/ + +TEST(string2Float, emptyString) +{ + ASSERT_EQ(string2Float(""), std::nullopt); +} + +TEST(string2Float, trivialConversions) +{ + ASSERT_EQ(string2Float("1.0"), 1.0); + + ASSERT_EQ(string2Float("0.0"), 0.0); + + ASSERT_EQ(string2Float("-100.25"), -100.25); +} + +/* ---------------------------------------------------------------------------- + * string2Int + * --------------------------------------------------------------------------*/ + +TEST(string2Int, emptyString) +{ + ASSERT_EQ(string2Int(""), std::nullopt); +} + +TEST(string2Int, trivialConversions) +{ + ASSERT_EQ(string2Int("1"), 1); + + ASSERT_EQ(string2Int("0"), 0); + + ASSERT_EQ(string2Int("-100"), -100); +} + +/* ---------------------------------------------------------------------------- + * renderSize + * --------------------------------------------------------------------------*/ + +TEST(renderSize, misc) +{ + ASSERT_EQ(renderSize(0, true), " 0.0 KiB"); + ASSERT_EQ(renderSize(100, true), " 0.1 KiB"); + ASSERT_EQ(renderSize(100), "0.1 KiB"); + ASSERT_EQ(renderSize(972, true), " 0.9 KiB"); + ASSERT_EQ(renderSize(973, true), " 1.0 KiB"); // FIXME: should round down + ASSERT_EQ(renderSize(1024, true), " 1.0 KiB"); + ASSERT_EQ(renderSize(1024 * 1024, true), "1024.0 KiB"); + ASSERT_EQ(renderSize(1100 * 1024, true), " 1.1 MiB"); + ASSERT_EQ(renderSize(2ULL * 1024 * 1024 * 1024, true), " 2.0 GiB"); + ASSERT_EQ(renderSize(2100ULL * 1024 * 1024 * 1024, true), " 2.1 TiB"); +} + +/* ---------------------------------------------------------------------------- + * rewriteStrings + * --------------------------------------------------------------------------*/ + +TEST(rewriteStrings, emptyString) +{ + StringMap rewrites; + rewrites["this"] = "that"; + + ASSERT_EQ(rewriteStrings("", rewrites), ""); +} + +TEST(rewriteStrings, emptyRewrites) +{ + StringMap rewrites; + + ASSERT_EQ(rewriteStrings("this and that", rewrites), "this and that"); +} + +TEST(rewriteStrings, successfulRewrite) +{ + StringMap rewrites; + rewrites["this"] = "that"; + + ASSERT_EQ(rewriteStrings("this and that", rewrites), "that and that"); +} + +TEST(rewriteStrings, doesntOccur) +{ + StringMap rewrites; + rewrites["foo"] = "bar"; + + ASSERT_EQ(rewriteStrings("this and that", rewrites), "this and that"); +} + +/* ---------------------------------------------------------------------------- + * replaceStrings + * --------------------------------------------------------------------------*/ + +TEST(replaceStrings, emptyString) +{ + ASSERT_EQ(replaceStrings("", "this", "that"), ""); + ASSERT_EQ(replaceStrings("this and that", "", ""), "this and that"); +} + +TEST(replaceStrings, successfulReplace) +{ + ASSERT_EQ(replaceStrings("this and that", "this", "that"), "that and that"); +} + +TEST(replaceStrings, doesntOccur) +{ + ASSERT_EQ(replaceStrings("this and that", "foo", "bar"), "this and that"); +} + +/* ---------------------------------------------------------------------------- + * trim + * --------------------------------------------------------------------------*/ + +TEST(trim, emptyString) +{ + ASSERT_EQ(trim(""), ""); +} + +TEST(trim, removesWhitespace) +{ + ASSERT_EQ(trim("foo"), "foo"); + ASSERT_EQ(trim(" foo "), "foo"); + ASSERT_EQ(trim(" foo bar baz"), "foo bar baz"); + ASSERT_EQ(trim(" \t foo bar baz\n"), "foo bar baz"); +} + +/* ---------------------------------------------------------------------------- + * chomp + * --------------------------------------------------------------------------*/ + +TEST(chomp, emptyString) +{ + ASSERT_EQ(chomp(""), ""); +} + +TEST(chomp, removesWhitespace) +{ + ASSERT_EQ(chomp("foo"), "foo"); + ASSERT_EQ(chomp("foo "), "foo"); + ASSERT_EQ(chomp(" foo "), " foo"); + ASSERT_EQ(chomp(" foo bar baz "), " foo bar baz"); + ASSERT_EQ(chomp("\t foo bar baz\n"), "\t foo bar baz"); +} + +/* ---------------------------------------------------------------------------- + * quoteStrings + * --------------------------------------------------------------------------*/ + +TEST(quoteStrings, empty) +{ + Strings s = {}; + Strings expected = {}; + + ASSERT_EQ(quoteStrings(s), expected); +} + +TEST(quoteStrings, emptyStrings) +{ + Strings s = {"", "", ""}; + Strings expected = {"''", "''", "''"}; + ASSERT_EQ(quoteStrings(s), expected); +} + +TEST(quoteStrings, trivialQuote) +{ + Strings s = {"foo", "bar", "baz"}; + Strings expected = {"'foo'", "'bar'", "'baz'"}; + + ASSERT_EQ(quoteStrings(s), expected); +} + +TEST(quoteStrings, quotedStrings) +{ + Strings s = {"'foo'", "'bar'", "'baz'"}; + Strings expected = {"''foo''", "''bar''", "''baz''"}; + + ASSERT_EQ(quoteStrings(s), expected); +} + +/* ---------------------------------------------------------------------------- + * get + * --------------------------------------------------------------------------*/ + +TEST(get, emptyContainer) +{ + StringMap s = {}; + auto expected = nullptr; + + ASSERT_EQ(get(s, "one"), expected); +} + +TEST(get, getFromContainer) +{ + StringMap s; + s["one"] = "yi"; + s["two"] = "er"; + auto expected = "yi"; + + ASSERT_EQ(*get(s, "one"), expected); +} + +TEST(getOr, emptyContainer) +{ + StringMap s = {}; + auto expected = "yi"; + + ASSERT_EQ(getOr(s, "one", "yi"), expected); +} + +TEST(getOr, getFromContainer) +{ + StringMap s; + s["one"] = "yi"; + s["two"] = "er"; + auto expected = "yi"; + + ASSERT_EQ(getOr(s, "one", "nope"), expected); +} + +} // namespace nix