diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fdd2d67f6..2aa3a3300 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,7 +64,7 @@ jobs: - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - uses: cachix/install-nix-action@v25 with: - install_url: https://releases.nixos.org/nix/nix-2.13.3/install + install_url: https://releases.nixos.org/nix/nix-2.20.3/install - uses: cachix/cachix-action@v14 with: name: '${{ env.CACHIX_NAME }}' @@ -116,7 +116,7 @@ jobs: fetch-depth: 0 - uses: cachix/install-nix-action@v25 with: - install_url: https://releases.nixos.org/nix/nix-2.13.3/install + install_url: https://releases.nixos.org/nix/nix-2.20.3/install - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#default.version | tr -d \")" >> $GITHUB_ENV - uses: cachix/cachix-action@v14 @@ -153,6 +153,8 @@ jobs: IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') docker tag nix:$NIX_VERSION $IMAGE_ID:$NIX_VERSION - docker tag nix:$NIX_VERSION $IMAGE_ID:master + docker tag nix:$NIX_VERSION $IMAGE_ID:latest docker push $IMAGE_ID:$NIX_VERSION + docker push $IMAGE_ID:latest + # deprecated 2024-02-24 docker push $IMAGE_ID:master diff --git a/.gitignore b/.gitignore index a47b195bb..5c1136823 100644 --- a/.gitignore +++ b/.gitignore @@ -45,13 +45,16 @@ perl/Makefile.config /src/libexpr/parser-tab.hh /src/libexpr/parser-tab.output /src/libexpr/nix.tbl +/src/libexpr/tests /tests/unit/libexpr/libnixexpr-tests # /src/libstore/ *.gen.* +/src/libstore/tests /tests/unit/libstore/libnixstore-tests # /src/libutil/ +/src/libutil/tests /tests/unit/libutil/libnixutil-tests /src/nix/nix @@ -94,6 +97,7 @@ perl/Makefile.config /tests/functional/ca/config.nix /tests/functional/dyn-drv/config.nix /tests/functional/repl-result-out +/tests/functional/debugger-test-out /tests/functional/test-libstoreconsumer/test-libstoreconsumer # /tests/functional/lang/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ffcc0268f..a0c2b16f4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -63,7 +63,7 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). - Functional tests – [`tests/functional/**.sh`](./tests/functional) - Unit tests – [`src/*/tests`](./src/) - Integration tests – [`tests/nixos/*`](./tests/nixos) - - [ ] User documentation in the [manual](..doc/manual/src) + - [ ] User documentation in the [manual](./doc/manual/src) - [ ] API documentation in header files - [ ] Code and comments are self-explanatory - [ ] Commit message explains **why** the change was made diff --git a/Makefile b/Makefile index e0f3359ef..af00e58b2 100644 --- a/Makefile +++ b/Makefile @@ -42,12 +42,24 @@ ifeq ($(ENABLE_FUNCTIONAL_TESTS), yes) makefiles += \ tests/functional/local.mk \ tests/functional/ca/local.mk \ + tests/functional/git-hashing/local.mk \ tests/functional/dyn-drv/local.mk \ tests/functional/local-overlay-store/local.mk \ tests/functional/test-libstoreconsumer/local.mk \ tests/functional/plugins/local.mk endif +# Some makefiles require access to built programs and must be included late. +makefiles-late = + +ifeq ($(ENABLE_DOC_GEN), yes) +makefiles-late += doc/manual/local.mk +endif + +ifeq ($(ENABLE_INTERNAL_API_DOCS), yes) +makefiles-late += doc/internal-api/local.mk +endif + # Miscellaneous global Flags OPTIMIZE = 1 @@ -57,6 +69,7 @@ ifeq ($(OPTIMIZE), 1) GLOBAL_LDFLAGS += $(CXXLTO) else GLOBAL_CXXFLAGS += -O0 -U_FORTIFY_SOURCE + unexport NIX_HARDENING_ENABLE endif include mk/platform.mk @@ -71,7 +84,7 @@ ifdef HOST_WINDOWS GLOBAL_LDFLAGS += -Wl,--export-all-symbols endif -GLOBAL_CXXFLAGS += -g -Wall -include $(buildprefix)config.h -std=c++2a -I src +GLOBAL_CXXFLAGS += -g -Wall -Wimplicit-fallthrough -include $(buildprefix)config.h -std=c++2a -I src # Include the main lib, causing rules to be defined @@ -96,24 +109,16 @@ installcheck: @exit 1 endif -# Documentation or else fallback stub rules. -# -# The documentation makefiles be included after `mk/lib.mk` so rules -# refer to variables defined by `mk/lib.mk`. Rules are not "lazy" like -# variables, unfortunately. +# Documentation fallback stub rules. -ifeq ($(ENABLE_DOC_GEN), yes) -$(eval $(call include-sub-makefile, doc/manual/local.mk)) -else +ifneq ($(ENABLE_DOC_GEN), yes) .PHONY: manual-html manpages manual-html manpages: @echo "Generated docs are disabled. Configure without '--disable-doc-gen', or avoid calling 'make manpages' and 'make manual-html'." @exit 1 endif -ifeq ($(ENABLE_INTERNAL_API_DOCS), yes) -$(eval $(call include-sub-makefile, doc/internal-api/local.mk)) -else +ifneq ($(ENABLE_INTERNAL_API_DOCS), yes) .PHONY: internal-api-html internal-api-html: @echo "Internal API docs are disabled. Configure with '--enable-internal-api-docs', or avoid calling 'make internal-api-html'." diff --git a/configure.ac b/configure.ac index 8c29c1e62..676b145a5 100644 --- a/configure.ac +++ b/configure.ac @@ -47,6 +47,10 @@ AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier ('cpu-os')]) # State should be stored in /nix/var, unless the user overrides it explicitly. test "$localstatedir" = '${prefix}/var' && localstatedir=/nix/var +# Assign a default value to C{,XX}FLAGS as the default configure script sets them +# to -O2 otherwise, which we don't want to have hardcoded +CFLAGS=${CFLAGS-""} +CXXFLAGS=${CXXFLAGS-""} AC_PROG_CC AC_PROG_CXX diff --git a/doc/manual/book.toml b/doc/manual/book.toml index 73fb7e75e..d524dbb13 100644 --- a/doc/manual/book.toml +++ b/doc/manual/book.toml @@ -6,6 +6,8 @@ additional-css = ["custom.css"] additional-js = ["redirects.js"] edit-url-template = "https://github.com/NixOS/nix/tree/master/doc/manual/{path}" git-repository-url = "https://github.com/NixOS/nix" +fold.enable = true +fold.level = 1 [preprocessor.anchors] renderers = ["html"] diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index d04f32b49..28b80f589 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -18,7 +18,7 @@ const redirects = { "chap-tuning-cores-and-jobs": "advanced-topics/cores-vs-jobs.html", "chap-diff-hook": "advanced-topics/diff-hook.html", "check-dirs-are-unregistered": "advanced-topics/diff-hook.html#check-dirs-are-unregistered", - "chap-distributed-builds": "advanced-topics/distributed-builds.html", + "chap-distributed-builds": "command-ref/conf-file.html#conf-builders", "chap-post-build-hook": "advanced-topics/post-build-hook.html", "chap-post-build-hook-caveats": "advanced-topics/post-build-hook.html#implementation-caveats", "chap-writing-nix-expressions": "language/index.html", @@ -358,7 +358,11 @@ const redirects = { "one-time-setup": "testing.html#one-time-setup", "using-the-ci-generated-installer-for-manual-testing": "testing.html#using-the-ci-generated-installer-for-manual-testing", "characterization-testing": "#characterisation-testing-unit", - } + }, + "glossary.html": { + "gloss-local-store": "store/types/local-store.html", + "gloss-chroot-store": "store/types/local-store.html", + }, }; // the following code matches the current page's URL against the set of redirects. diff --git a/doc/manual/rl-next/better-errors-in-nix-repl.md b/doc/manual/rl-next/better-errors-in-nix-repl.md new file mode 100644 index 000000000..4deaa8c70 --- /dev/null +++ b/doc/manual/rl-next/better-errors-in-nix-repl.md @@ -0,0 +1,40 @@ +--- +synopsis: Concise error printing in `nix repl` +prs: 9928 +--- + +Previously, if an element of a list or attribute set threw an error while +evaluating, `nix repl` would print the entire error (including source location +information) inline. This output was clumsy and difficult to parse: + +``` +nix-repl> { err = builtins.throw "uh oh!"; } +{ err = «error: + … while calling the 'throw' builtin + at «string»:1:9: + 1| { err = builtins.throw "uh oh!"; } + | ^ + + error: uh oh!»; } +``` + +Now, only the error message is displayed, making the output much more readable. +``` +nix-repl> { err = builtins.throw "uh oh!"; } +{ err = «error: uh oh!»; } +``` + +However, if the whole expression being evaluated throws an error, source +locations and (if applicable) a stack trace are printed, just like you'd expect: + +``` +nix-repl> builtins.throw "uh oh!" +error: + … while calling the 'throw' builtin + at «string»:1:1: + 1| builtins.throw "uh oh!" + | ^ + + error: uh oh! +``` + diff --git a/doc/manual/rl-next/debugger-locals-for-let-expressions.md b/doc/manual/rl-next/debugger-locals-for-let-expressions.md new file mode 100644 index 000000000..736208724 --- /dev/null +++ b/doc/manual/rl-next/debugger-locals-for-let-expressions.md @@ -0,0 +1,9 @@ +--- +synopsis: "`--debugger` can now access bindings from `let` expressions" +prs: 9918 +issues: 8827. +--- + +Breakpoints and errors in the bindings of a `let` expression can now access +those bindings in the debugger. Previously, only the body of `let` expressions +could access those bindings. diff --git a/doc/manual/rl-next/debugger-on-trace.md b/doc/manual/rl-next/debugger-on-trace.md new file mode 100644 index 000000000..721928550 --- /dev/null +++ b/doc/manual/rl-next/debugger-on-trace.md @@ -0,0 +1,9 @@ +--- +synopsis: Enter the `--debugger` when `builtins.trace` is called if `debugger-on-trace` is set +prs: 9914 +--- + +If the `debugger-on-trace` option is set and `--debugger` is given, +`builtins.trace` calls will behave similarly to `builtins.break` and will enter +the debug REPL. This is useful for determining where warnings are being emitted +from. diff --git a/doc/manual/rl-next/debugger-positions.md b/doc/manual/rl-next/debugger-positions.md new file mode 100644 index 000000000..2fe868413 --- /dev/null +++ b/doc/manual/rl-next/debugger-positions.md @@ -0,0 +1,25 @@ +--- +synopsis: Debugger prints source position information +prs: 9913 +--- + +The `--debugger` now prints source location information, instead of the +pointers of source location information. Before: + +``` +nix-repl> :bt +0: while evaluating the attribute 'python311.pythonForBuild.pkgs' +0x600001522598 +``` + +After: + +``` +0: while evaluating the attribute 'python311.pythonForBuild.pkgs' +/nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in +``` diff --git a/doc/manual/rl-next/enter-debugger-more-reliably-in-let-and-calls.md b/doc/manual/rl-next/enter-debugger-more-reliably-in-let-and-calls.md new file mode 100644 index 000000000..c93225816 --- /dev/null +++ b/doc/manual/rl-next/enter-debugger-more-reliably-in-let-and-calls.md @@ -0,0 +1,25 @@ +--- +synopsis: The `--debugger` will start more reliably in `let` expressions and function calls +prs: 9917 +issues: 6649 +--- + +Previously, if you attempted to evaluate this file with the debugger: + +```nix +let + a = builtins.trace "before inner break" ( + builtins.break "hello" + ); + b = builtins.trace "before outer break" ( + builtins.break a + ); +in + b +``` + +Nix would correctly enter the debugger at `builtins.break a`, but if you asked +it to `:continue`, it would skip over the `builtins.break "hello"` expression +entirely. + +Now, Nix will correctly enter the debugger at both breakpoints. diff --git a/doc/manual/rl-next/inherit-from-by-need.md b/doc/manual/rl-next/inherit-from-by-need.md new file mode 100644 index 000000000..67c2cdedf --- /dev/null +++ b/doc/manual/rl-next/inherit-from-by-need.md @@ -0,0 +1,7 @@ +--- +synopsis: "`inherit (x) ...` evaluates `x` only once" +prs: 9847 +--- + +`inherit (x) a b ...` now evaluates the expression `x` only once for all inherited attributes rather than once for each inherited attribute. +This does not usually have a measurable impact, but side-effects (such as `builtins.trace`) would be duplicated and expensive expressions (such as derivations) could cause a measurable slowdown. diff --git a/doc/manual/rl-next/lambda-printing.md b/doc/manual/rl-next/lambda-printing.md new file mode 100644 index 000000000..3a63f3068 --- /dev/null +++ b/doc/manual/rl-next/lambda-printing.md @@ -0,0 +1,50 @@ +--- +synopsis: Functions are printed with more detail +prs: 9606 +issues: 7145 +--- + +Functions and `builtins` are printed with more detail in `nix repl`, `nix +eval`, `builtins.trace`, and most other places values are printed. + +Before: + +``` +$ nix repl nixpkgs +nix-repl> builtins.map +«primop» + +nix-repl> builtins.map lib.id +«primop-app» + +nix-repl> builtins.trace lib.id "my-value" +trace: +"my-value" + +$ nix eval --file functions.nix +{ id = ; primop = ; primop-app = ; } +``` + +After: + +``` +$ nix repl nixpkgs +nix-repl> builtins.map +«primop map» + +nix-repl> builtins.map lib.id +«partially applied primop map» + +nix-repl> builtins.trace lib.id "my-value" +trace: «lambda id @ /nix/store/8rrzq23h2zq7sv5l2vhw44kls5w0f654-source/lib/trivial.nix:26:5» +"my-value" + +$ nix eval --file functions.nix +{ id = «lambda id @ /Users/wiggles/nix/functions.nix:2:8»; primop = «primop map»; primop-app = «partially applied primop map»; } +``` + +This was actually released in Nix 2.20, but wasn't added to the release notes +so we're announcing it here. The historical release notes have been updated as well. + +[type-error]: https://github.com/NixOS/nix/pull/9753 +[coercion-error]: https://github.com/NixOS/nix/pull/9754 diff --git a/doc/manual/rl-next/more-commands-respect-ctrl-c.md b/doc/manual/rl-next/more-commands-respect-ctrl-c.md new file mode 100644 index 000000000..948930c96 --- /dev/null +++ b/doc/manual/rl-next/more-commands-respect-ctrl-c.md @@ -0,0 +1,13 @@ +--- +synopsis: Nix commands respect Ctrl-C +prs: 9687 6995 +issues: 7245 +--- + +Previously, many Nix commands would hang indefinitely if Ctrl-C was pressed +while performing various operations (including `nix develop`, `nix flake +update`, and so on). With several fixes to Nix's signal handlers, Nix commands +will now exit quickly after Ctrl-C is pressed. + +This was actually released in Nix 2.20, but wasn't added to the release notes +so we're announcing it here. The historical release notes have been updated as well. diff --git a/doc/manual/rl-next/pretty-print-in-nix-repl.md b/doc/manual/rl-next/pretty-print-in-nix-repl.md new file mode 100644 index 000000000..26ba5162a --- /dev/null +++ b/doc/manual/rl-next/pretty-print-in-nix-repl.md @@ -0,0 +1,24 @@ +--- +synopsis: "`nix repl` pretty-prints values" +prs: 9931 +--- + +`nix repl` will now pretty-print values: + +``` +{ + attrs = { + a = { + b = { + c = { }; + }; + }; + }; + list = [ 1 ]; + list' = [ + 1 + 2 + 3 + ]; +} +``` diff --git a/doc/manual/rl-next/reduce-debugger-clutter.md b/doc/manual/rl-next/reduce-debugger-clutter.md new file mode 100644 index 000000000..9bc902eee --- /dev/null +++ b/doc/manual/rl-next/reduce-debugger-clutter.md @@ -0,0 +1,37 @@ +--- +synopsis: "Visual clutter in `--debugger` is reduced" +prs: 9919 +--- + +Before: +``` +info: breakpoint reached + + +Starting REPL to allow you to inspect the current state of the evaluator. + +Welcome to Nix 2.20.0pre20231222_dirty. Type :? for help. + +nix-repl> :continue +error: uh oh + + +Starting REPL to allow you to inspect the current state of the evaluator. + +Welcome to Nix 2.20.0pre20231222_dirty. Type :? for help. + +nix-repl> +``` + +After: + +``` +info: breakpoint reached + +Nix 2.20.0pre20231222_dirty debugger +Type :? for help. +nix-repl> :continue +error: uh oh + +nix-repl> +``` diff --git a/doc/manual/rl-next/repl-ctrl-c-while-printing.md b/doc/manual/rl-next/repl-ctrl-c-while-printing.md new file mode 100644 index 000000000..15b0daa0a --- /dev/null +++ b/doc/manual/rl-next/repl-ctrl-c-while-printing.md @@ -0,0 +1,8 @@ +--- +synopsis: "`nix repl` now respects Ctrl-C while printing values" +prs: 9927 +--- + +`nix repl` will now halt immediately when Ctrl-C is pressed while it's printing +a value. This is useful if you got curious about what would happen if you +printed all of Nixpkgs. diff --git a/doc/manual/rl-next/repl-cycle-detection.md b/doc/manual/rl-next/repl-cycle-detection.md new file mode 100644 index 000000000..de24c4be1 --- /dev/null +++ b/doc/manual/rl-next/repl-cycle-detection.md @@ -0,0 +1,22 @@ +--- +synopsis: Cycle detection in `nix repl` is simpler and more reliable +prs: 9926 +issues: 8672 +--- + +The cycle detection in `nix repl`, `nix eval`, `builtins.trace`, and everywhere +else values are printed is now simpler and matches the cycle detection in +`nix-instantiate --eval` output. + +Before: + +``` +nix eval --expr 'let self = { inherit self; }; in self' +{ self = { self = «repeated»; }; } +``` + +After: + +``` +{ self = «repeated»; } +``` diff --git a/doc/manual/rl-next/source-location-in-while-evaluating-attribute.md b/doc/manual/rl-next/source-location-in-while-evaluating-attribute.md new file mode 100644 index 000000000..0e0b74c5a --- /dev/null +++ b/doc/manual/rl-next/source-location-in-while-evaluating-attribute.md @@ -0,0 +1,23 @@ +--- +synopsis: "In the debugger, `while evaluating the attribute` errors now include position information" +prs: 9915 +--- + +Before: + +``` +0: while evaluating the attribute 'python311.pythonForBuild.pkgs' +0x600001522598 +``` + +After: + +``` +0: while evaluating the attribute 'python311.pythonForBuild.pkgs' +/nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in +``` diff --git a/doc/manual/rl-next/stack-size-macos.md b/doc/manual/rl-next/stack-size-macos.md new file mode 100644 index 000000000..b1c40bb5a --- /dev/null +++ b/doc/manual/rl-next/stack-size-macos.md @@ -0,0 +1,9 @@ +--- +synopsis: Stack size is increased on macOS +prs: 9860 +--- + +Previously, Nix would set the stack size to 64MiB on Linux, but would leave the +stack size set to the default (approximately 8KiB) on macOS. Now, the stack +size is correctly set to 64MiB on macOS as well, which should reduce stack +overflow segfaults in deeply-recursive Nix expressions. diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 695d63dfc..70dea4fbd 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -104,11 +104,12 @@ - [Channels](command-ref/files/channels.md) - [Default Nix expression](command-ref/files/default-nix-expression.md) - [Architecture and Design](architecture/architecture.md) -- [JSON Formats](json/index.md) - - [Store Object Info](json/store-object-info.md) - - [Derivation](json/derivation.md) -- [Protocols](protocols/index.md) +- [Formats and Protocols](protocols/index.md) + - [JSON Formats](protocols/json/index.md) + - [Store Object Info](protocols/json/store-object-info.md) + - [Derivation](protocols/json/derivation.md) - [Serving Tarball Flakes](protocols/tarball-fetcher.md) + - [Store Path Specification](protocols/store-path.md) - [Derivation "ATerm" file format](protocols/derivation-aterm.md) - [Glossary](glossary.md) - [Contributing](contributing/index.md) diff --git a/doc/manual/_redirects b/doc/manual/src/_redirects similarity index 97% rename from doc/manual/_redirects rename to doc/manual/src/_redirects index 62c693c97..8bf0e854b 100644 --- a/doc/manual/_redirects +++ b/doc/manual/src/_redirects @@ -36,5 +36,6 @@ /package-management/s3-substituter /store/types/s3-binary-cache-store 301! /protocols/protocols /protocols 301! +/json/* /protocols/json/:splat 301! /release-notes/release-notes /release-notes 301! diff --git a/doc/manual/src/advanced-topics/distributed-builds.md b/doc/manual/src/advanced-topics/distributed-builds.md index 507c5ecb7..52acd039c 100644 --- a/doc/manual/src/advanced-topics/distributed-builds.md +++ b/doc/manual/src/advanced-topics/distributed-builds.md @@ -36,16 +36,8 @@ error: cannot connect to 'mac' then you need to ensure that the `PATH` of non-interactive login shells contains Nix. -> **Warning** -> -> If you are building via the Nix daemon, it is the Nix daemon user account (that is, `root`) that should have SSH access to a user (not necessarily `root`) on the remote machine. -> -> If you can’t or don’t want to configure `root` to be able to access the remote machine, you can use a private Nix store instead by passing e.g. `--store ~/my-nix` when running a Nix command from the local machine. - -The list of remote machines can be specified on the command line or in -the Nix configuration file. The former is convenient for testing. For -example, the following command allows you to build a derivation for -`x86_64-darwin` on a Linux machine: +The [list of remote build machines](@docroot@/command-ref/conf-file.md#conf-builders) can be specified on the command line or in the Nix configuration file. +For example, the following command allows you to build a derivation for `x86_64-darwin` on a Linux machine: ```console $ uname @@ -60,97 +52,20 @@ $ cat ./result Darwin ``` -It is possible to specify multiple builders separated by a semicolon or -a newline, e.g. +It is possible to specify multiple build machines separated by a semicolon or a newline, e.g. ```console --builders 'ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd' ``` -Each machine specification consists of the following elements, separated -by spaces. Only the first element is required. To leave a field at its -default, set it to `-`. - -1. The URI of the remote store in the format - `ssh://[username@]hostname`, e.g. `ssh://nix@mac` or `ssh://mac`. - For backward compatibility, `ssh://` may be omitted. The hostname - may be an alias defined in your `~/.ssh/config`. - -2. A comma-separated list of Nix platform type identifiers, such as - `x86_64-darwin`. It is possible for a machine to support multiple - platform types, e.g., `i686-linux,x86_64-linux`. If omitted, this - defaults to the local platform type. - -3. The SSH identity file to be used to log in to the remote machine. If - omitted, SSH will use its regular identities. - -4. The maximum number of builds that Nix will execute in parallel on - the machine. Typically this should be equal to the number of CPU - cores. For instance, the machine `itchy` in the example will execute - up to 8 builds in parallel. - -5. The “speed factor”, indicating the relative speed of the machine. If - there are multiple machines of the right type, Nix will prefer the - fastest, taking load into account. - -6. A comma-separated list of *supported features*. If a derivation has - the `requiredSystemFeatures` attribute, then Nix will only perform - the derivation on a machine that has the specified features. For - instance, the attribute - - ```nix - requiredSystemFeatures = [ "kvm" ]; - ``` - - will cause the build to be performed on a machine that has the `kvm` - feature. - -7. A comma-separated list of *mandatory features*. A machine will only - be used to build a derivation if all of the machine’s mandatory - features appear in the derivation’s `requiredSystemFeatures` - attribute. - -8. The (base64-encoded) public host key of the remote machine. If omitted, SSH - will use its regular known-hosts file. Specifically, the field is calculated - via `base64 -w0 /etc/ssh/ssh_host_ed25519_key.pub`. - -For example, the machine specification - - nix@scratchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 8 1 kvm - nix@itchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 8 2 - nix@poochie.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 1 2 kvm benchmark - -specifies several machines that can perform `i686-linux` builds. -However, `poochie` will only do builds that have the attribute - -```nix -requiredSystemFeatures = [ "benchmark" ]; -``` - -or - -```nix -requiredSystemFeatures = [ "benchmark" "kvm" ]; -``` - -`itchy` cannot do builds that require `kvm`, but `scratchy` does support -such builds. For regular builds, `itchy` will be preferred over -`scratchy` because it has a higher speed factor. - -Remote builders can also be configured in `nix.conf`, e.g. +Remote build machines can also be configured in [`nix.conf`](@docroot@/command-ref/conf-file.md), e.g. builders = ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd -Finally, remote builders can be configured in a separate configuration -file included in `builders` via the syntax `@file`. For example, +Finally, remote build machines can be configured in a separate configuration +file included in `builders` via the syntax `@/path/to/file`. For example, builders = @/etc/nix/machines -causes the list of machines in `/etc/nix/machines` to be included. (This -is the default.) - -If you want the builders to use caches, you likely want to set the -option `builders-use-substitutes` in your local `nix.conf`. - -To build only on remote builders and disable building on the local -machine, you can use the option `--max-jobs 0`. +causes the list of machines in `/etc/nix/machines` to be included. +(This is the default.) diff --git a/doc/manual/src/command-ref/nix-collect-garbage.md b/doc/manual/src/command-ref/nix-collect-garbage.md index 3cab79f0e..1bc88d858 100644 --- a/doc/manual/src/command-ref/nix-collect-garbage.md +++ b/doc/manual/src/command-ref/nix-collect-garbage.md @@ -51,7 +51,7 @@ These options are for deleting old [profiles] prior to deleting unreachable [sto - [`--delete-old`](#opt-delete-old) / `-d`\ Delete all old generations of profiles. - This is the equivalent of invoking `nix-env --delete-generations old` on each found profile. + This is the equivalent of invoking [`nix-env --delete-generations old`](@docroot@/command-ref/nix-env/delete-generations.md#generations-old) on each found profile. - [`--delete-older-than`](#opt-delete-older-than) *period*\ Delete all generations of profiles older than the specified amount (except for the generations that were active at that point in time). diff --git a/doc/manual/src/command-ref/nix-env/delete-generations.md b/doc/manual/src/command-ref/nix-env/delete-generations.md index adc6fc219..6b6ea798e 100644 --- a/doc/manual/src/command-ref/nix-env/delete-generations.md +++ b/doc/manual/src/command-ref/nix-env/delete-generations.md @@ -12,13 +12,13 @@ This operation deletes the specified generations of the current profile. *generations* can be a one of the following: -- `...`:\ +- [`...`](#generations-list):\ A list of generation numbers, each one a separate command-line argument. Delete exactly the profile generations given by their generation number. Deleting the current generation is not allowed. -- The special value `old` +- [The special value `old`](#generations-old) Delete all generations except the current one. @@ -30,7 +30,7 @@ This operation deletes the specified generations of the current profile. > Because one can roll back to a previous generation, it is possible to have generations newer than the current one. > They will also be deleted. -- `d`:\ +- [`d`](#generations-time):\ The last *number* days *Example*: `30d` @@ -38,7 +38,7 @@ This operation deletes the specified generations of the current profile. Delete all generations created more than *number* days ago, except the most recent one of them. This allows rolling back to generations that were available within the specified period. -- `+`:\ +- [`+`](#generations-count):\ The last *number* generations up to the present *Example*: `+5` diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 9e2470859..916ec3077 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -44,13 +44,13 @@ To build Nix itself in this shell: ```console [nix-shell]$ autoreconfPhase [nix-shell]$ configurePhase -[nix-shell]$ make -j $NIX_BUILD_CORES +[nix-shell]$ make -j $NIX_BUILD_CORES OPTIMIZE=0 ``` To install it in `$(pwd)/outputs` and test it: ```console -[nix-shell]$ make install +[nix-shell]$ make install OPTIMIZE=0 [nix-shell]$ make installcheck check -j $NIX_BUILD_CORES [nix-shell]$ nix --version nix (Nix) 2.12 @@ -147,10 +147,10 @@ Nix can be built for various platforms, as specified in [`flake.nix`]: In order to build Nix for a different platform than the one you're currently on, you need a way for your current Nix installation to build code for that -platform. Common solutions include [remote builders] and [binary format emulation] +platform. Common solutions include [remote build machines] and [binary format emulation] (only supported on NixOS). -[remote builders]: ../advanced-topics/distributed-builds.md +[remote builders]: @docroot@/language/derivations.md#attr-builder [binary format emulation]: https://nixos.org/manual/nixos/stable/options.html#opt-boot.binfmt.emulatedSystems Given such a setup, executing the build only requires selecting the respective attribute. diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 13b2906f7..a71b2e2b3 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -37,7 +37,7 @@ This can be achieved by: - Fetching a pre-built [store object] from a [substituter] - Running the [`builder`](@docroot@/language/derivations.md#attr-builder) executable as specified in the corresponding [derivation] - - Delegating to a [remote builder](@docroot@/advanced-topics/distributed-builds.html) and retrieving the outputs + - Delegating to a [remote machine](@docroot@/command-ref/conf-file.md#conf-builders) and retrieving the outputs See [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md) for a detailed description of the algorithm. @@ -59,23 +59,12 @@ - [store]{#gloss-store} - A collection of store objects, with operations to manipulate that collection. - See [Nix store](./store/index.md) for details. + A collection of [store objects][store object], with operations to manipulate that collection. + See [Nix Store](./store/index.md) for details. - There are many types of stores. - See [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md) for a complete list. - - From the perspective of the location where Nix is invoked, the Nix store can be referred to _local_ or _remote_. - Only a [local store]{#gloss-local-store} exposes a location in the file system of the machine where Nix is invoked that allows access to store objects, typically `/nix/store`. - Local stores can be used for building [derivations](#gloss-derivation). - See [Local Store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) for details. + There are many types of stores, see [Store Types](./store/types/index.md) for details. [store]: #gloss-store - [local store]: #gloss-local-store - -- [chroot store]{#gloss-chroot-store} - - A [local store] whose canonical path is anything other than `/nix/store`. - [binary cache]{#gloss-binary-cache} @@ -87,7 +76,7 @@ - [store path]{#gloss-store-path} - The location of a [store object](@docroot@/store/index.md#store-object) in the file system, i.e., an immediate child of the Nix store directory. + The location of a [store object] in the file system, i.e., an immediate child of the Nix store directory. > **Example** > @@ -243,6 +232,7 @@ - All paths in the store path's [closure] are valid. [validity]: #gloss-validity + [local store]: @docroot@/store/types/local-store.md - [user environment]{#gloss-user-env} diff --git a/doc/manual/src/installation/upgrading.md b/doc/manual/src/installation/upgrading.md index 47618e2f5..38edcdbc5 100644 --- a/doc/manual/src/installation/upgrading.md +++ b/doc/manual/src/installation/upgrading.md @@ -16,7 +16,7 @@ nix (Nix) 2.18.1 > Writing to the [local store](@docroot@/store/types/local-store.md) with a newer version of Nix, for example by building derivations with [`nix-build`](@docroot@/command-ref/nix-build.md) or [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md), may change the database schema! > Reverting to an older version of Nix may therefore require purging the store database before it can be used. -### Linux multi-user +## Linux multi-user ```console $ sudo su diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 5a6c00cd4..7306fc182 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -257,7 +257,7 @@ Derivations can declare some infrequently used optional attributes. of the environment (typically, a few hundred kilobyte). - [`preferLocalBuild`]{#adv-attr-preferLocalBuild}\ - If this attribute is set to `true` and [distributed building is enabled](../advanced-topics/distributed-builds.md), then, if possible, the derivation will be built locally instead of being forwarded to a remote machine. + If this attribute is set to `true` and [distributed building is enabled](@docroot@/command-ref/conf-file.md#conf-builders), then, if possible, the derivation will be built locally instead of being forwarded to a remote machine. This is useful for derivations that are cheapest to build locally. - [`allowSubstitutes`]{#adv-attr-allowSubstitutes}\ diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index cbb30d074..75f824a34 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -36,7 +36,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect The system type on which the [`builder`](#attr-builder) executable is meant to be run. A necessary condition for Nix to build derivations locally is that the `system` attribute matches the current [`system` configuration option]. - It can automatically [build on other platforms](../advanced-topics/distributed-builds.md) by forwarding build requests to other machines. + It can automatically [build on other platforms](@docroot@/language/derivations.md#attr-builder) by forwarding build requests to other machines. [`system` configuration option]: @docroot@/command-ref/conf-file.md#conf-system diff --git a/doc/manual/src/language/import-from-derivation.md b/doc/manual/src/language/import-from-derivation.md index 03b3f9d91..fb12ba51a 100644 --- a/doc/manual/src/language/import-from-derivation.md +++ b/doc/manual/src/language/import-from-derivation.md @@ -1,6 +1,8 @@ # Import From Derivation -The value of a Nix expression can depend on the contents of a [store object](@docroot@/glossary.md#gloss-store-object). +The value of a Nix expression can depend on the contents of a [store object]. + +[store object]: @docroot@/glossary.md#gloss-store-object Passing an expression `expr` that evaluates to a [store path](@docroot@/glossary.md#gloss-store-path) to any built-in function which reads from the filesystem constitutes Import From Derivation (IFD): diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index e9cbb5c92..6fd66864b 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -84,7 +84,7 @@ The `+` operator is overloaded to also work on strings and paths. > > *string* `+` *string* -Concatenate two [string]s and merge their string contexts. +Concatenate two [strings][string] and merge their string contexts. [String concatenation]: #string-concatenation @@ -94,7 +94,7 @@ Concatenate two [string]s and merge their string contexts. > > *path* `+` *path* -Concatenate two [path]s. +Concatenate two [paths][path]. The result is a path. [Path concatenation]: #path-concatenation @@ -150,9 +150,9 @@ If an attribute name is present in both, the attribute value from the latter is Comparison is -- [arithmetic] for [number]s -- lexicographic for [string]s and [path]s -- item-wise lexicographic for [list]s: +- [arithmetic] for [numbers][number] +- lexicographic for [strings][string] and [paths][path] +- item-wise lexicographic for [lists][list]: elements at the same index in both lists are compared according to their type and skipped if they are equal. All comparison operators are implemented in terms of `<`, and the following equivalencies hold: @@ -163,12 +163,12 @@ All comparison operators are implemented in terms of `<`, and the following equi | *a* `>` *b* | *b* `<` *a* | | *a* `>=` *b* | `! (` *a* `<` *b* `)` | -[Comparison]: #comparison-operators +[Comparison]: #comparison ## Equality -- [Attribute sets][attribute set] and [list]s are compared recursively, and therefore are fully evaluated. -- Comparison of [function]s always returns `false`. +- [Attribute sets][attribute set] and [lists][list] are compared recursively, and therefore are fully evaluated. +- Comparison of [functions][function] always returns `false`. - Numbers are type-compatible, see [arithmetic] operators. - Floating point numbers only differ up to a limited precision. diff --git a/doc/manual/src/language/string-interpolation.md b/doc/manual/src/language/string-interpolation.md index 6e28d2664..7d81c2020 100644 --- a/doc/manual/src/language/string-interpolation.md +++ b/doc/manual/src/language/string-interpolation.md @@ -20,6 +20,8 @@ Rather than writing (where `freetype` is a [derivation]), you can instead write +[derivation]: ../glossary.md#gloss-derivation + ```nix "--with-freetype2-library=${freetype}/lib" ``` diff --git a/doc/manual/src/language/values.md b/doc/manual/src/language/values.md index aea68a441..74ffc7070 100644 --- a/doc/manual/src/language/values.md +++ b/doc/manual/src/language/values.md @@ -156,6 +156,8 @@ function and the fifth being a set. Note that lists are only lazy in values, and they are strict in length. +Elements in a list can be accessed using [`builtins.elemAt`](./builtins.md#builtins-elemAt). + ## Attribute Set An attribute set is a collection of name-value-pairs (called *attributes*) enclosed in curly brackets (`{ }`). diff --git a/doc/manual/src/json/derivation.md b/doc/manual/src/protocols/json/derivation.md similarity index 100% rename from doc/manual/src/json/derivation.md rename to doc/manual/src/protocols/json/derivation.md diff --git a/doc/manual/src/json/store-object-info.md b/doc/manual/src/protocols/json/store-object-info.md similarity index 96% rename from doc/manual/src/json/store-object-info.md rename to doc/manual/src/protocols/json/store-object-info.md index db43c2fa1..ba4ab098f 100644 --- a/doc/manual/src/json/store-object-info.md +++ b/doc/manual/src/protocols/json/store-object-info.md @@ -14,11 +14,11 @@ Info about a [store object]. * `narHash`: - Hash of the [file system object] part of the store object when serialized as a [Nix Archive](#gloss-nar). + Hash of the [file system object] part of the store object when serialized as a [Nix Archive]. * `narSize`: - Size of the [file system object] part of the store object when serialized as a [Nix Archive](#gloss-nar). + Size of the [file system object] part of the store object when serialized as a [Nix Archive]. * `references`: @@ -30,6 +30,7 @@ Info about a [store object]. [store path]: @docroot@/glossary.md#gloss-store-path [file system object]: @docroot@/store/file-system-object.md +[Nix Archive]: @docroot@/glossary.md#gloss-nar ## Impure fields diff --git a/doc/manual/src/protocols/store-path.md b/doc/manual/src/protocols/store-path.md new file mode 100644 index 000000000..565c4fa75 --- /dev/null +++ b/doc/manual/src/protocols/store-path.md @@ -0,0 +1,131 @@ +# Complete Store Path Calculation + +This is the complete specification for how store paths are calculated. + +The format of this specification is close to [Extended Backus–Naur form](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form), but must deviate for a few things such as hash functions which we treat as bidirectional for specification purposes. + +Regular users do *not* need to know this information --- store paths can be treated as black boxes computed from the properties of the store objects they refer to. +But for those interested in exactly how Nix works, e.g. if they are reimplementing it, this information can be useful. + +## Store path proper + +```ebnf +store-path = store-dir "/" digest "-" name +``` +where + +- `name` = the name of the store object. + +- `store-dir` = the [store directory](@docroot@/store/store-path.md#store-directory) + +- `digest` = base-32 representation of the first 160 bits of a [SHA-256] hash of `fingerprint` + + This the hash part of the store name + +## Fingerprint + +- ```ebnf + fingerprint = type ":" sha256 ":" inner-digest ":" store ":" name + ``` + + Note that it includes the location of the store as well as the name to make sure that changes to either of those are reflected in the hash + (e.g. you won't get `/nix/store/-name1` and `/nix/store/-name2`, or `/gnu/store/-name1`, with equal hash parts). + +- `type` = one of: + + - ```ebnf + | "text" ( ":" store-path )* + ``` + + for encoded derivations written to the store. + The optional trailing store paths are the references of the store object. + + - ```ebnf + | "source" ( ":" store-path )* + ``` + + For paths copied to the store and hashed via a [Nix Archive (NAR)] and [SHA-256][sha-256]. + Just like in the text case, we can have the store objects referenced by their paths. + Additionally, we can have an optional `:self` label to denote self reference. + + - ```ebnf + | "output:" id + ``` + + For either the outputs built from derivations, + paths copied to the store hashed that area single file hashed directly, or the via a hash algorithm other than [SHA-256][sha-256]. + (in that case "source" is used; this is only necessary for compatibility). + + `id` is the name of the output (usually, "out"). + For content-addressed store objects, `id`, is always "out". + +- `inner-digest` = base-16 representation of a SHA-256 hash of `inner-fingerprint` + +## Inner fingerprint + +- `inner-fingerprint` = one of the following based on `type`: + + - if `type` = `"text:" ...`: + + the string written to the resulting store path. + + - if `type` = `"source:" ...`: + + the the hash of the [Nix Archive (NAR)] serialization of the [file system object](@docroot@/store/file-system-object.md) of the store object. + + - if `type` = `"output:" id`: + + - For input-addressed derivation outputs: + + the [ATerm](@docroot@/protocols/derivation-aterm.md) serialization of the derivation modulo fixed output derivations. + + - For content-addressed store paths: + + ```ebnf + "fixed:out:" rec algo ":" hash ":" + ``` + + where + + - `rec` = one of: + + - ```ebnf + | "" + ``` + (empty string) for hashes of the flat (single file) serialization + + - ```ebnf + | "r:" + ``` + hashes of the for [Nix Archive (NAR)] (arbitrary file system object) serialization + + - ```ebnf + | "git:" + ``` + hashes of the [Git blob/tree](https://git-scm.com/book/en/v2/Git-Internals-Git-Objects) [Merkel tree](https://en.wikipedia.org/wiki/Merkle_tree) format + + - ```ebnf + algo = "md5" | "sha1" | "sha256" + ``` + + - `hash` = base-16 representation of the path or flat hash of the contents of the path (or expected contents of the path for fixed-output derivations). + + Note that `id` = `"out"`, regardless of the name part of the store path. + Also note that NAR + SHA-256 must not use this case, and instead must use the `type` = `"source:" ...` case. + +[Nix Archive (NAR)]: @docroot@/glossary.md#gloss-NAR +[sha-256]: https://en.m.wikipedia.org/wiki/SHA-256 + +### Historical Note + +The `type` = `"source:" ...` and `type` = `"output:out"` grammars technically overlap in purpose, +in that both can represent data hashed by its SHA-256 NAR serialization. + +The original reason for this way of computing names was to prevent name collisions (for security). +For instance, the thinking was that it shouldn't be feasible to come up with a derivation whose output path collides with the path for a copied source. +The former would have an `inner-fingerprint` starting with `output:out:`, while the latter would have an `inner-fingerprint` starting with `source:`. + +Since `64519cfd657d024ae6e2bb74cb21ad21b886fd2a` (2008), however, it was decided that separating derivation-produced vs manually-hashed content-addressed data like this was not useful. +Now, data that is content-addressed with SHA-256 + NAR-serialization always uses the `source:...` construction, regardless of how it was produced (manually or by derivation). +This allows freely switching between using [fixed-output derivations](@docroot@/glossary.md#gloss-fixed-output-derivation) for fetching, and fetching out-of-band and then manually adding. +It also removes the ambiguity from the grammar. diff --git a/doc/manual/src/release-notes/rl-2.20.md b/doc/manual/src/release-notes/rl-2.20.md index 26869e90a..8ede168a4 100644 --- a/doc/manual/src/release-notes/rl-2.20.md +++ b/doc/manual/src/release-notes/rl-2.20.md @@ -167,3 +167,36 @@ error: expected a set but found an integer ``` + +- Functions are printed with more detail [#7145](https://github.com/NixOS/nix/issues/7145) [#9606](https://github.com/NixOS/nix/pull/9606) + + `nix repl`, `nix eval`, `builtins.trace`, and most other places values are + printed will now include function names and source location information: + + ``` + $ nix repl nixpkgs + nix-repl> builtins.map + «primop map» + + nix-repl> builtins.map lib.id + «partially applied primop map» + + nix-repl> builtins.trace lib.id "my-value" + trace: «lambda id @ /nix/store/8rrzq23h2zq7sv5l2vhw44kls5w0f654-source/lib/trivial.nix:26:5» + "my-value" + ``` + +- Flake operations like `nix develop` will no longer fail when run in a Git + repository where the `flake.lock` file is `.gitignore`d + [#8854](https://github.com/NixOS/nix/issues/8854) + [#9324](https://github.com/NixOS/nix/pull/9324) + +- Nix commands will now respect Ctrl-C + [#7145](https://github.com/NixOS/nix/issues/7145) + [#6995](https://github.com/NixOS/nix/pull/6995) + [#9687](https://github.com/NixOS/nix/pull/9687) + + Previously, many Nix commands would hang indefinitely if Ctrl-C was pressed + while performing various operations (including `nix develop`, `nix flake + update`, and so on). With several fixes to Nix's signal handlers, Nix + commands will now exit quickly after Ctrl-C is pressed. diff --git a/flake.lock b/flake.lock index f0efb4036..bb2e400c0 100644 --- a/flake.lock +++ b/flake.lock @@ -34,16 +34,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1705033721, - "narHash": "sha256-K5eJHmL1/kev6WuqyqqbS1cdNnSidIZ3jeqJ7GbrYnQ=", + "lastModified": 1709083642, + "narHash": "sha256-7kkJQd4rZ+vFrzWu8sTRtta5D1kBG0LSRYAfhtmMlSo=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "a1982c92d8980a0114372973cbdfe0a307f1bdea", + "rev": "b550fe4b4776908ac2a861124307045f8e717c8e", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-23.05-small", + "ref": "release-23.11", "repo": "nixpkgs", "type": "github" } diff --git a/flake.nix b/flake.nix index 0bc70768e..42aaace67 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,9 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; + # TODO switch to nixos-23.11-small + # https://nixpk.gs/pr-tracker.html?pr=291954 + inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-23.11"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; }; @@ -10,20 +12,10 @@ let inherit (nixpkgs) lib; - - # Experimental fileset library: https://github.com/NixOS/nixpkgs/pull/222981 - # Not an "idiomatic" flake input because: - # - Propagation to dependent locks: https://github.com/NixOS/nix/issues/7730 - # - Subflake would download redundant and huge parent flake - # - No git tree hash support: https://github.com/NixOS/nix/issues/6044 - inherit (import (builtins.fetchTarball { url = "https://github.com/NixOS/nix/archive/1bdcd7fc8a6a40b2e805bad759b36e64e911036b.tar.gz"; sha256 = "sha256:14ljlpdsp4x7h1fkhbmc4bd3vsqnx8zdql4h3037wh09ad6a0893"; })) - fileset; + inherit (lib) fileset; officialRelease = false; - # Set to true to build the release notes for the next release. - buildUnreleasedNotes = false; - version = lib.fileContents ./.version + versionSuffix; versionSuffix = if officialRelease @@ -404,8 +396,11 @@ # Make bash completion work. XDG_DATA_DIRS+=:$out/share ''; + nativeBuildInputs = attrs.nativeBuildInputs or [] - ++ lib.optional stdenv.cc.isClang pkgs.buildPackages.bear + # TODO: Remove the darwin check once + # https://github.com/NixOS/nixpkgs/pull/291814 is available + ++ lib.optional (stdenv.cc.isClang && !stdenv.buildPlatform.isDarwin) pkgs.buildPackages.bear ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) pkgs.buildPackages.clang-tools; }); in @@ -417,8 +412,9 @@ (forAllStdenvs (stdenvName: makeShell pkgs pkgs.${stdenvName})); in (makeShells "native" nixpkgsFor.${system}.native) // - (makeShells "static" nixpkgsFor.${system}.static) // - (lib.genAttrs shellCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) // + (lib.optionalAttrs (!nixpkgsFor.${system}.native.stdenv.isDarwin) + (makeShells "static" nixpkgsFor.${system}.static)) // + (lib.genAttrs shellCrossSystems (crossSystem: let pkgs = nixpkgsFor.${system}.cross.${crossSystem}; in makeShell pkgs pkgs.stdenv)) // { default = self.devShells.${system}.native-stdenvPackages; } diff --git a/mk/lib.mk b/mk/lib.mk index 10ce8d436..fe0add1c9 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -97,6 +97,10 @@ $(foreach test-group, $(install-tests-groups), \ $(eval $(call run-test,$(test),$(install_test_init))) \ $(eval $(test-group).test-group: $(test).test))) +# Include makefiles requiring built programs. +$(foreach mf, $(makefiles-late), $(eval $(call include-sub-makefile,$(mf)))) + + $(foreach file, $(man-pages), $(eval $(call install-data-in, $(file), $(mandir)/man$(patsubst .%,%,$(suffix $(file)))))) diff --git a/mk/templates.mk b/mk/templates.mk index 866bdc17f..d5dae61c7 100644 --- a/mk/templates.mk +++ b/mk/templates.mk @@ -10,10 +10,10 @@ endef ifneq ($(MAKECMDGOALS), clean) -$(buildprefix)%.h: %.h.in +$(buildprefix)%.h: %.h.in $(buildprefix)config.status $(trace-gen) rm -f $@ && cd $(buildprefixrel) && ./config.status --quiet --header=$(@:$(buildprefix)%=%) -$(buildprefix)%: %.in +$(buildprefix)%: %.in $(buildprefix)config.status $(trace-gen) rm -f $@ && cd $(buildprefixrel) && ./config.status --quiet --file=$(@:$(buildprefix)%=%) endif diff --git a/package.nix b/package.nix index d1d14d10e..20796a386 100644 --- a/package.nix +++ b/package.nix @@ -154,7 +154,7 @@ in { in fileset.toSource { root = ./.; - fileset = fileset.intersect baseFiles (fileset.unions ([ + fileset = fileset.intersection baseFiles (fileset.unions ([ # For configure ./.version ./configure.ac @@ -209,6 +209,10 @@ in { (lib.getBin lowdown) mdbook mdbook-linkcheck + ] ++ lib.optionals doInstallCheck [ + git + mercurial + openssh ] ++ lib.optionals (doInstallCheck || enableManual) [ jq # Also for custom mdBook preprocessor. ] ++ lib.optional stdenv.hostPlatform.isLinux util-linux @@ -249,12 +253,6 @@ in { dontBuild = !attrs.doBuild; doCheck = attrs.doCheck; - nativeCheckInputs = [ - git - mercurial - openssh - ]; - disallowedReferences = [ boost ]; preConfigure = lib.optionalString (doBuild && ! stdenv.hostPlatform.isStatic) ( diff --git a/perl/.yath.rc b/perl/.yath.rc new file mode 100644 index 000000000..118bf80c8 --- /dev/null +++ b/perl/.yath.rc @@ -0,0 +1,2 @@ +[test] +-I=rel(lib/Nix) diff --git a/perl/default.nix b/perl/default.nix index 4687976a1..7103574c9 100644 --- a/perl/default.nix +++ b/perl/default.nix @@ -5,12 +5,12 @@ , nix, curl, bzip2, xz, boost, libsodium, darwin }: -perl.pkgs.toPerlModule (stdenv.mkDerivation { +perl.pkgs.toPerlModule (stdenv.mkDerivation (finalAttrs: { name = "nix-perl-${nix.version}"; src = fileset.toSource { root = ../.; - fileset = fileset.unions [ + fileset = fileset.unions ([ ../.version ../m4 ../mk @@ -20,7 +20,10 @@ perl.pkgs.toPerlModule (stdenv.mkDerivation { ./configure.ac ./lib ./local.mk - ]; + ] ++ lib.optionals finalAttrs.doCheck [ + ./.yath.rc + ./t + ]); }; nativeBuildInputs = @@ -40,6 +43,13 @@ perl.pkgs.toPerlModule (stdenv.mkDerivation { ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security; + # `perlPackages.Test2Harness` is marked broken for Darwin + doCheck = !stdenv.isDarwin; + + nativeCheckInputs = [ + perlPackages.Test2Harness + ]; + configureFlags = [ "--with-dbi=${perlPackages.DBI}/${perl.libPrefix}" "--with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}" @@ -48,4 +58,4 @@ perl.pkgs.toPerlModule (stdenv.mkDerivation { enableParallelBuilding = true; postUnpack = "sourceRoot=$sourceRoot/perl"; -}) +})) diff --git a/perl/lib/Nix/Store.pm b/perl/lib/Nix/Store.pm index 3e4bbee0a..16f2e17c8 100644 --- a/perl/lib/Nix/Store.pm +++ b/perl/lib/Nix/Store.pm @@ -12,17 +12,20 @@ our %EXPORT_TAGS = ( 'all' => [ qw( ) ] ); our @EXPORT_OK = ( @{ $EXPORT_TAGS{'all'} } ); our @EXPORT = qw( - setVerbosity - isValidPath queryReferences queryPathInfo queryDeriver queryPathHash - queryPathFromHashPart - topoSortPaths computeFSClosure followLinksToStorePath exportPaths importPaths + StoreWrapper + StoreWrapper::new + StoreWrapper::isValidPath StoreWrapper::queryReferences StoreWrapper::queryPathInfo StoreWrapper::queryDeriver StoreWrapper::queryPathHash + StoreWrapper::queryPathFromHashPart + StoreWrapper::topoSortPaths StoreWrapper::computeFSClosure followLinksToStorePath StoreWrapper::exportPaths StoreWrapper::importPaths + StoreWrapper::addToStore StoreWrapper::makeFixedOutputPath + StoreWrapper::derivationFromPath + StoreWrapper::addTempRoot + StoreWrapper::queryRawRealisation + hashPath hashFile hashString convertHash signString checkSignature - addToStore makeFixedOutputPath - derivationFromPath - addTempRoot getBinDir getStoreDir - queryRawRealisation + setVerbosity ); our $VERSION = '0.15'; diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 423c01cf7..1c64cc66b 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -17,47 +17,61 @@ #include #include - using namespace nix; +static bool libStoreInitialized = false; -static ref store() -{ - static std::shared_ptr _store; - if (!_store) { - try { - initLibStore(); - _store = openStore(); - } catch (Error & e) { - croak("%s", e.what()); - } - } - return ref(_store); -} - +struct StoreWrapper { + ref store; +}; MODULE = Nix::Store PACKAGE = Nix::Store PROTOTYPES: ENABLE +TYPEMAP: < _store; try { - RETVAL = store()->isValidPath(store()->parseStorePath(path)); + if (!libStoreInitialized) { + initLibStore(); + libStoreInitialized = true; + } + if (items == 1) { + _store = openStore(); + RETVAL = new StoreWrapper { + .store = ref{_store} + }; + } else { + RETVAL = new StoreWrapper { + .store = openStore(s) + }; + } } catch (Error & e) { croak("%s", e.what()); } @@ -65,52 +79,81 @@ int isValidPath(char * path) RETVAL -SV * queryReferences(char * path) +void init() + CODE: + if (!libStoreInitialized) { + initLibStore(); + libStoreInitialized = true; + } + + +void setVerbosity(int level) + CODE: + verbosity = (Verbosity) level; + + +int +StoreWrapper::isValidPath(char * path) + CODE: + try { + RETVAL = THIS->store->isValidPath(THIS->store->parseStorePath(path)); + } catch (Error & e) { + croak("%s", e.what()); + } + OUTPUT: + RETVAL + + +SV * +StoreWrapper::queryReferences(char * path) PPCODE: try { - for (auto & i : store()->queryPathInfo(store()->parseStorePath(path))->references) - XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(i).c_str(), 0))); + for (auto & i : THIS->store->queryPathInfo(THIS->store->parseStorePath(path))->references) + XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(i).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); } -SV * queryPathHash(char * path) +SV * +StoreWrapper::queryPathHash(char * path) PPCODE: try { - auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(HashFormat::Nix32, true); + auto s = THIS->store->queryPathInfo(THIS->store->parseStorePath(path))->narHash.to_string(HashFormat::Nix32, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); } -SV * queryDeriver(char * path) +SV * +StoreWrapper::queryDeriver(char * path) PPCODE: try { - auto info = store()->queryPathInfo(store()->parseStorePath(path)); + auto info = THIS->store->queryPathInfo(THIS->store->parseStorePath(path)); if (!info->deriver) XSRETURN_UNDEF; - XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0))); + XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(*info->deriver).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); } -SV * queryPathInfo(char * path, int base32) +SV * +StoreWrapper::queryPathInfo(char * path, int base32) PPCODE: try { - auto info = store()->queryPathInfo(store()->parseStorePath(path)); + auto info = THIS->store->queryPathInfo(THIS->store->parseStorePath(path)); if (!info->deriver) XPUSHs(&PL_sv_undef); else - XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0))); + XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(*info->deriver).c_str(), 0))); auto s = info->narHash.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); AV * refs = newAV(); for (auto & i : info->references) - av_push(refs, newSVpv(store()->printStorePath(i).c_str(), 0)); + av_push(refs, newSVpv(THIS->store->printStorePath(i).c_str(), 0)); XPUSHs(sv_2mortal(newRV((SV *) refs))); AV * sigs = newAV(); for (auto & i : info->sigs) @@ -120,10 +163,11 @@ SV * queryPathInfo(char * path, int base32) croak("%s", e.what()); } -SV * queryRawRealisation(char * outputId) +SV * +StoreWrapper::queryRawRealisation(char * outputId) PPCODE: try { - auto realisation = store()->queryRealisation(DrvOutput::parse(outputId)); + auto realisation = THIS->store->queryRealisation(DrvOutput::parse(outputId)); if (realisation) XPUSHs(sv_2mortal(newSVpv(realisation->toJSON().dump().c_str(), 0))); else @@ -133,46 +177,50 @@ SV * queryRawRealisation(char * outputId) } -SV * queryPathFromHashPart(char * hashPart) +SV * +StoreWrapper::queryPathFromHashPart(char * hashPart) PPCODE: try { - auto path = store()->queryPathFromHashPart(hashPart); - XPUSHs(sv_2mortal(newSVpv(path ? store()->printStorePath(*path).c_str() : "", 0))); + auto path = THIS->store->queryPathFromHashPart(hashPart); + XPUSHs(sv_2mortal(newSVpv(path ? THIS->store->printStorePath(*path).c_str() : "", 0))); } catch (Error & e) { croak("%s", e.what()); } -SV * computeFSClosure(int flipDirection, int includeOutputs, ...) +SV * +StoreWrapper::computeFSClosure(int flipDirection, int includeOutputs, ...) PPCODE: try { StorePathSet paths; for (int n = 2; n < items; ++n) - store()->computeFSClosure(store()->parseStorePath(SvPV_nolen(ST(n))), paths, flipDirection, includeOutputs); + THIS->store->computeFSClosure(THIS->store->parseStorePath(SvPV_nolen(ST(n))), paths, flipDirection, includeOutputs); for (auto & i : paths) - XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(i).c_str(), 0))); + XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(i).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); } -SV * topoSortPaths(...) +SV * +StoreWrapper::topoSortPaths(...) PPCODE: try { StorePathSet paths; - for (int n = 0; n < items; ++n) paths.insert(store()->parseStorePath(SvPV_nolen(ST(n)))); - auto sorted = store()->topoSortPaths(paths); + for (int n = 0; n < items; ++n) paths.insert(THIS->store->parseStorePath(SvPV_nolen(ST(n)))); + auto sorted = THIS->store->topoSortPaths(paths); for (auto & i : sorted) - XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(i).c_str(), 0))); + XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(i).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); } -SV * followLinksToStorePath(char * path) +SV * +StoreWrapper::followLinksToStorePath(char * path) CODE: try { - RETVAL = newSVpv(store()->printStorePath(store()->followLinksToStorePath(path)).c_str(), 0); + RETVAL = newSVpv(THIS->store->printStorePath(THIS->store->followLinksToStorePath(path)).c_str(), 0); } catch (Error & e) { croak("%s", e.what()); } @@ -180,35 +228,38 @@ SV * followLinksToStorePath(char * path) RETVAL -void exportPaths(int fd, ...) +void +StoreWrapper::exportPaths(int fd, ...) PPCODE: try { StorePathSet paths; - for (int n = 1; n < items; ++n) paths.insert(store()->parseStorePath(SvPV_nolen(ST(n)))); + for (int n = 1; n < items; ++n) paths.insert(THIS->store->parseStorePath(SvPV_nolen(ST(n)))); FdSink sink(fd); - store()->exportPaths(paths, sink); + THIS->store->exportPaths(paths, sink); } catch (Error & e) { croak("%s", e.what()); } -void importPaths(int fd, int dontCheckSigs) +void +StoreWrapper::importPaths(int fd, int dontCheckSigs) PPCODE: try { FdSource source(fd); - store()->importPaths(source, dontCheckSigs ? NoCheckSigs : CheckSigs); + THIS->store->importPaths(source, dontCheckSigs ? NoCheckSigs : CheckSigs); } catch (Error & e) { croak("%s", e.what()); } -SV * hashPath(char * algo, int base32, char * path) +SV * +hashPath(char * algo, int base32, char * path) PPCODE: try { - PosixSourceAccessor accessor; + auto [accessor, canonPath] = PosixSourceAccessor::createAtRoot(path); Hash h = hashPath( - accessor, CanonPath::fromCwd(path), - FileIngestionMethod::Recursive, parseHashAlgo(algo)).first; + accessor, canonPath, + FileIngestionMethod::Recursive, parseHashAlgo(algo)); auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { @@ -280,64 +331,67 @@ int checkSignature(SV * publicKey_, SV * sig_, char * msg) RETVAL -SV * addToStore(char * srcPath, int recursive, char * algo) +SV * +StoreWrapper::addToStore(char * srcPath, int recursive, char * algo) PPCODE: try { auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - PosixSourceAccessor accessor; - auto path = store()->addToStore( + auto [accessor, canonPath] = PosixSourceAccessor::createAtRoot(srcPath); + auto path = THIS->store->addToStore( std::string(baseNameOf(srcPath)), - accessor, CanonPath::fromCwd(srcPath), + accessor, canonPath, method, parseHashAlgo(algo)); - XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); + XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(path).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); } -SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) +SV * +StoreWrapper::makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) PPCODE: try { auto h = Hash::parseAny(hash, parseHashAlgo(algo)); auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - auto path = store()->makeFixedOutputPath(name, FixedOutputInfo { + auto path = THIS->store->makeFixedOutputPath(name, FixedOutputInfo { .method = method, .hash = h, .references = {}, }); - XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); + XPUSHs(sv_2mortal(newSVpv(THIS->store->printStorePath(path).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); } -SV * derivationFromPath(char * drvPath) +SV * +StoreWrapper::derivationFromPath(char * drvPath) PREINIT: HV *hash; CODE: try { - Derivation drv = store()->derivationFromPath(store()->parseStorePath(drvPath)); + Derivation drv = THIS->store->derivationFromPath(THIS->store->parseStorePath(drvPath)); hash = newHV(); HV * outputs = newHV(); - for (auto & i : drv.outputsAndOptPaths(*store())) { + for (auto & i : drv.outputsAndOptPaths(*THIS->store)) { hv_store( outputs, i.first.c_str(), i.first.size(), !i.second.second ? newSV(0) /* null value */ - : newSVpv(store()->printStorePath(*i.second.second).c_str(), 0), + : newSVpv(THIS->store->printStorePath(*i.second.second).c_str(), 0), 0); } hv_stores(hash, "outputs", newRV((SV *) outputs)); AV * inputDrvs = newAV(); for (auto & i : drv.inputDrvs.map) - av_push(inputDrvs, newSVpv(store()->printStorePath(i.first).c_str(), 0)); // !!! ignores i->second + av_push(inputDrvs, newSVpv(THIS->store->printStorePath(i.first).c_str(), 0)); // !!! ignores i->second hv_stores(hash, "inputDrvs", newRV((SV *) inputDrvs)); AV * inputSrcs = newAV(); for (auto & i : drv.inputSrcs) - av_push(inputSrcs, newSVpv(store()->printStorePath(i).c_str(), 0)); + av_push(inputSrcs, newSVpv(THIS->store->printStorePath(i).c_str(), 0)); hv_stores(hash, "inputSrcs", newRV((SV *) inputSrcs)); hv_stores(hash, "platform", newSVpv(drv.platform.c_str(), 0)); @@ -361,10 +415,11 @@ SV * derivationFromPath(char * drvPath) RETVAL -void addTempRoot(char * storePath) +void +StoreWrapper::addTempRoot(char * storePath) PPCODE: try { - store()->addTempRoot(store()->parseStorePath(storePath)); + THIS->store->addTempRoot(THIS->store->parseStorePath(storePath)); } catch (Error & e) { croak("%s", e.what()); } diff --git a/perl/local.mk b/perl/local.mk index 0eae651d8..ed4764eb9 100644 --- a/perl/local.mk +++ b/perl/local.mk @@ -41,3 +41,6 @@ Store_FORCE_INSTALL = 1 Store_INSTALL_DIR = $(perllibdir)/auto/Nix/Store clean-files += lib/Nix/Config.pm lib/Nix/Store.cc Makefile.config + +check: all + yath test diff --git a/perl/t/init.t b/perl/t/init.t new file mode 100644 index 000000000..80197e013 --- /dev/null +++ b/perl/t/init.t @@ -0,0 +1,13 @@ +use strict; +use warnings; +use Test2::V0; + +use Nix::Store; + +my $s = new Nix::Store("dummy://"); + +my $res = $s->isValidPath("/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"); + +ok(!$res, "should not have path"); + +done_testing; diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh index 766f81bde..24c9052f9 100644 --- a/scripts/install-darwin-multi-user.sh +++ b/scripts/install-darwin-multi-user.sh @@ -102,7 +102,7 @@ poly_extra_try_me_commands() { poly_configure_nix_daemon_service() { task "Setting up the nix-daemon LaunchDaemon" _sudo "to set up the nix-daemon as a LaunchDaemon" \ - /usr/bin/install -m -rw-r--r-- "/nix/var/nix/profiles/default$NIX_DAEMON_DEST" "$NIX_DAEMON_DEST" + /usr/bin/install -m "u=rw,go=r" "/nix/var/nix/profiles/default$NIX_DAEMON_DEST" "$NIX_DAEMON_DEST" _sudo "to load the LaunchDaemon plist for nix-daemon" \ launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index ad3ee8881..1dbb93bf9 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -58,6 +58,31 @@ readonly EXTRACTED_NIX_PATH="$(dirname "$0")" readonly ROOT_HOME=~root +readonly PROXY_ENVIRONMENT_VARIABLES=( + http_proxy + https_proxy + ftp_proxy + no_proxy + HTTP_PROXY + HTTPS_PROXY + FTP_PROXY + NO_PROXY +) + +SUDO_EXTRA_ENVIRONMENT_VARIABLES=() + +setup_sudo_extra_environment_variables() { + local i=${#SUDO_EXTRA_ENVIRONMENT_VARIABLES[@]} + for variable in "${PROXY_ENVIRONMENT_VARIABLES[@]}"; do + if [ "x${!variable:-}" != "x" ]; then + SUDO_EXTRA_ENVIRONMENT_VARIABLES[i]="$variable=${!variable}" + i=$((i + 1)) + fi + done +} + +setup_sudo_extra_environment_variables + if [ -t 0 ] && [ -z "${NIX_INSTALLER_YES:-}" ]; then readonly IS_HEADLESS='no' else @@ -361,7 +386,7 @@ _sudo() { if is_root; then env "$@" else - sudo "$@" + sudo "${SUDO_EXTRA_ENVIRONMENT_VARIABLES[@]}" "$@" fi } diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 519e03242..118468477 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -202,7 +202,7 @@ static int main_build_remote(int argc, char * * argv) else drvstr = ""; - auto error = hintformat(errorText); + auto error = HintFmt(errorText); error % drvstr % neededSystem diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 193972272..444ff81c9 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -9,6 +9,7 @@ #include "store-api.hh" #include "command.hh" #include "tarball.hh" +#include "fetch-to-store.hh" namespace nix { @@ -156,7 +157,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) for (auto & i : autoArgs) { auto v = state.allocValue(); if (i.second[0] == 'E') - state.mkThunk_(*v, state.parseExprFromString(i.second.substr(1), state.rootPath(CanonPath::fromCwd()))); + state.mkThunk_(*v, state.parseExprFromString(i.second.substr(1), state.rootPath("."))); else v->mkString(((std::string_view) i.second).substr(1)); res.insert(state.symbols.create(i.first), v); @@ -164,11 +165,12 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) return res.finish(); } -SourcePath lookupFileArg(EvalState & state, std::string_view s, CanonPath baseDir) +SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * baseDir) { if (EvalSettings::isPseudoUrl(s)) { - auto storePath = fetchers::downloadTarball( - state.store, EvalSettings::resolvePseudoUrl(s), "source", false).storePath; + auto accessor = fetchers::downloadTarball( + EvalSettings::resolvePseudoUrl(s)).accessor; + auto storePath = fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy); return state.rootPath(CanonPath(state.store->toRealPath(storePath))); } @@ -185,7 +187,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, CanonPath baseDi } else - return state.rootPath(CanonPath(s, baseDir)); + return state.rootPath(baseDir ? absPath(s, *baseDir) : absPath(s)); } } diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index 4b403d936..2eb63e15d 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -29,6 +29,6 @@ private: std::map autoArgs; }; -SourcePath lookupFileArg(EvalState & state, std::string_view s, CanonPath baseDir = CanonPath::fromCwd()); +SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * baseDir = nullptr); } diff --git a/src/libcmd/editor-for.cc b/src/libcmd/editor-for.cc index 67653d9c9..6bf36bd64 100644 --- a/src/libcmd/editor-for.cc +++ b/src/libcmd/editor-for.cc @@ -17,7 +17,7 @@ Strings editorFor(const SourcePath & file, uint32_t line) editor.find("vim") != std::string::npos || editor.find("kak") != std::string::npos)) args.push_back(fmt("+%d", line)); - args.push_back(path->abs()); + args.push_back(path->string()); return args; } diff --git a/src/libcmd/installable-value.cc b/src/libcmd/installable-value.cc index c8a3e1b21..1aa2e65c1 100644 --- a/src/libcmd/installable-value.cc +++ b/src/libcmd/installable-value.cc @@ -45,7 +45,7 @@ ref InstallableValue::require(ref installable) std::optional InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx) { if (v.type() == nPath) { - auto storePath = fetchToStore(*state->store, v.path()); + auto storePath = fetchToStore(*state->store, v.path(), FetchMode::Copy); return {{ .path = DerivedPath::Opaque { .path = std::move(storePath), diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 736c41a1e..16d25d3cf 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -487,10 +487,11 @@ Installables SourceExprCommand::parseInstallables( state->eval(e, *vFile); } else if (file) { - state->evalFile(lookupFileArg(*state, *file, CanonPath::fromCwd(getCommandBaseDir())), *vFile); + auto dir = absPath(getCommandBaseDir()); + state->evalFile(lookupFileArg(*state, *file, &dir), *vFile); } else { - CanonPath dir(CanonPath::fromCwd(getCommandBaseDir())); + Path dir = absPath(getCommandBaseDir()); auto e = state->parseExprFromString(*expr, state->rootPath(dir)); state->eval(e, *vFile); } diff --git a/src/libcmd/misc-store-flags.cc b/src/libcmd/misc-store-flags.cc new file mode 100644 index 000000000..e66d3f63b --- /dev/null +++ b/src/libcmd/misc-store-flags.cc @@ -0,0 +1,121 @@ +#include "misc-store-flags.hh" + +namespace nix::flag +{ + +static void hashFormatCompleter(AddCompletions & completions, size_t index, std::string_view prefix) +{ + for (auto & format : hashFormats) { + if (hasPrefix(format, prefix)) { + completions.add(format); + } + } +} + +Args::Flag hashFormatWithDefault(std::string && longName, HashFormat * hf) +{ + assert(*hf == nix::HashFormat::SRI); + return Args::Flag { + .longName = std::move(longName), + .description = "Hash format (`base16`, `nix32`, `base64`, `sri`). Default: `sri`.", + .labels = {"hash-format"}, + .handler = {[hf](std::string s) { + *hf = parseHashFormat(s); + }}, + .completer = hashFormatCompleter, + }; +} + +Args::Flag hashFormatOpt(std::string && longName, std::optional * ohf) +{ + return Args::Flag { + .longName = std::move(longName), + .description = "Hash format (`base16`, `nix32`, `base64`, `sri`).", + .labels = {"hash-format"}, + .handler = {[ohf](std::string s) { + *ohf = std::optional{parseHashFormat(s)}; + }}, + .completer = hashFormatCompleter, + }; +} + +static void hashAlgoCompleter(AddCompletions & completions, size_t index, std::string_view prefix) +{ + for (auto & algo : hashAlgorithms) + if (hasPrefix(algo, prefix)) + completions.add(algo); +} + +Args::Flag hashAlgo(std::string && longName, HashAlgorithm * ha) +{ + return Args::Flag { + .longName = std::move(longName), + .description = "Hash algorithm (`md5`, `sha1`, `sha256`, or `sha512`).", + .labels = {"hash-algo"}, + .handler = {[ha](std::string s) { + *ha = parseHashAlgo(s); + }}, + .completer = hashAlgoCompleter, + }; +} + +Args::Flag hashAlgoOpt(std::string && longName, std::optional * oha) +{ + return Args::Flag { + .longName = std::move(longName), + .description = "Hash algorithm (`md5`, `sha1`, `sha256`, or `sha512`). Can be omitted for SRI hashes.", + .labels = {"hash-algo"}, + .handler = {[oha](std::string s) { + *oha = std::optional{parseHashAlgo(s)}; + }}, + .completer = hashAlgoCompleter, + }; +} + +Args::Flag fileIngestionMethod(FileIngestionMethod * method) +{ + return Args::Flag { + .longName = "mode", + // FIXME indentation carefully made for context, this is messed up. + .description = R"( + How to compute the hash of the input. + One of: + + - `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function. + + - `flat`: Assumes that the input is a single file and directly passes it to the hash function; + )", + .labels = {"file-ingestion-method"}, + .handler = {[method](std::string s) { + *method = parseFileIngestionMethod(s); + }}, + }; +} + +Args::Flag contentAddressMethod(ContentAddressMethod * method) +{ + return Args::Flag { + .longName = "mode", + // FIXME indentation carefully made for context, this is messed up. + .description = R"( + How to compute the content-address of the store object. + One of: + + - `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function. + + - `flat`: Assumes that the input is a single file and directly passes it to the hash function; + + - `text`: Like `flat`, but used for + [derivations](@docroot@/glossary.md#store-derivation) serialized in store object and + [`builtins.toFile`](@docroot@/language/builtins.html#builtins-toFile). + For advanced use-cases only; + for regular usage prefer `nar` and `flat. + )", + .labels = {"content-address-method"}, + .handler = {[method](std::string s) { + *method = ContentAddressMethod::parse(s); + }}, + }; +} + +} diff --git a/src/libcmd/misc-store-flags.hh b/src/libcmd/misc-store-flags.hh new file mode 100644 index 000000000..124372af7 --- /dev/null +++ b/src/libcmd/misc-store-flags.hh @@ -0,0 +1,21 @@ +#include "args.hh" +#include "content-address.hh" + +namespace nix::flag { + +Args::Flag hashAlgo(std::string && longName, HashAlgorithm * ha); +static inline Args::Flag hashAlgo(HashAlgorithm * ha) +{ + return hashAlgo("hash-algo", ha); +} +Args::Flag hashAlgoOpt(std::string && longName, std::optional * oha); +Args::Flag hashFormatWithDefault(std::string && longName, HashFormat * hf); +Args::Flag hashFormatOpt(std::string && longName, std::optional * ohf); +static inline Args::Flag hashAlgoOpt(std::optional * oha) +{ + return hashAlgoOpt("hash-algo", oha); +} +Args::Flag fileIngestionMethod(FileIngestionMethod * method); +Args::Flag contentAddressMethod(ContentAddressMethod * method); + +} diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index d7d8f9819..8b83608fa 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -52,6 +52,27 @@ extern "C" { namespace nix { +/** + * Returned by `NixRepl::processLine`. + */ +enum class ProcessLineResult { + /** + * The user exited with `:quit`. The REPL should exit. The surrounding + * program or evaluation (e.g., if the REPL was acting as the debugger) + * should also exit. + */ + Quit, + /** + * The user exited with `:continue`. The REPL should exit, but the program + * should continue running. + */ + Continue, + /** + * The user did not exit. The REPL should request another line of input. + */ + PromptAgain, +}; + struct NixRepl : AbstractNixRepl #if HAVE_BOEHMGC @@ -75,13 +96,13 @@ struct NixRepl std::function getValues); virtual ~NixRepl(); - void mainLoop() override; + ReplExitStatus mainLoop() override; void initEnv() override; StringSet completePrefix(const std::string & prefix); bool getLine(std::string & input, const std::string & prompt); StorePath getDerivationPath(Value & v); - bool processLine(std::string line); + ProcessLineResult processLine(std::string line); void loadFile(const Path & path); void loadFlake(const std::string & flakeRef); @@ -101,7 +122,8 @@ struct NixRepl .ansiColors = true, .force = true, .derivationPaths = true, - .maxDepth = maxDepth + .maxDepth = maxDepth, + .prettyIndent = 2 }); } }; @@ -232,7 +254,7 @@ static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positi : positions[dt.expr.getPos() ? dt.expr.getPos() : noPos]; if (pos) { - out << pos; + out << *pos; if (auto loc = pos->getCodeLines()) { out << "\n"; printCodeLines(out, "", *pos, *loc); @@ -243,10 +265,19 @@ static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positi return out; } -void NixRepl::mainLoop() +static bool isFirstRepl = true; + +ReplExitStatus NixRepl::mainLoop() { - std::string error = ANSI_RED "error:" ANSI_NORMAL " "; - notice("Welcome to Nix " + nixVersion + ". Type :? for help.\n"); + if (isFirstRepl) { + std::string_view debuggerNotice = ""; + if (state->debugRepl) { + debuggerNotice = " debugger"; + } + notice("Nix %1%%2%\nType :? for help.", nixVersion, debuggerNotice); + } + + isFirstRepl = false; loadFiles(); @@ -277,15 +308,25 @@ void NixRepl::mainLoop() // When continuing input from previous lines, don't print a prompt, just align to the same // number of chars as the prompt. if (!getLine(input, input.empty() ? "nix-repl> " : " ")) { - // ctrl-D should exit the debugger. + // Ctrl-D should exit the debugger. state->debugStop = false; - state->debugQuit = true; logger->cout(""); - break; + // TODO: Should Ctrl-D exit just the current debugger session or + // the entire program? + return ReplExitStatus::QuitAll; } logger->resume(); try { - if (!removeWhitespace(input).empty() && !processLine(input)) return; + switch (processLine(input)) { + case ProcessLineResult::Quit: + return ReplExitStatus::QuitAll; + case ProcessLineResult::Continue: + return ReplExitStatus::Continue; + case ProcessLineResult::PromptAgain: + break; + default: + abort(); + } } catch (ParseError & e) { if (e.msg().find("unexpected end of file") != std::string::npos) { // For parse errors on incomplete input, we continue waiting for the next line of @@ -342,7 +383,6 @@ bool NixRepl::getLine(std::string & input, const std::string & prompt) }; setupSignals(); - Finally resetTerminal([&]() { rl_deprep_terminal(); }); char * s = readline(prompt.c_str()); Finally doFree([&]() { free(s); }); restoreSignals(); @@ -422,8 +462,6 @@ StringSet NixRepl::completePrefix(const std::string & prefix) // Quietly ignore parse errors. } catch (EvalError & e) { // Quietly ignore evaluation errors. - } catch (UndefinedVarError & e) { - // Quietly ignore undefined variable errors. } catch (BadURL & e) { // Quietly ignore BadURL flake-related errors. } @@ -475,10 +513,11 @@ void NixRepl::loadDebugTraceEnv(DebugTrace & dt) } } -bool NixRepl::processLine(std::string line) +ProcessLineResult NixRepl::processLine(std::string line) { line = trim(line); - if (line == "") return true; + if (line.empty()) + return ProcessLineResult::PromptAgain; _isInterrupted = false; @@ -573,13 +612,13 @@ bool NixRepl::processLine(std::string line) else if (state->debugRepl && (command == ":s" || command == ":step")) { // set flag to stop at next DebugTrace; exit repl. state->debugStop = true; - return false; + return ProcessLineResult::Continue; } else if (state->debugRepl && (command == ":c" || command == ":continue")) { // set flag to run to next breakpoint or end of program; exit repl. state->debugStop = false; - return false; + return ProcessLineResult::Continue; } else if (command == ":a" || command == ":add") { @@ -722,8 +761,7 @@ bool NixRepl::processLine(std::string line) else if (command == ":q" || command == ":quit") { state->debugStop = false; - state->debugQuit = true; - return false; + return ProcessLineResult::Quit; } else if (command == ":doc") { @@ -784,7 +822,7 @@ bool NixRepl::processLine(std::string line) } } - return true; + return ProcessLineResult::PromptAgain; } void NixRepl::loadFile(const Path & path) @@ -890,7 +928,7 @@ void NixRepl::addVarToScope(const Symbol name, Value & v) Expr * NixRepl::parseString(std::string s) { - return state->parseExprFromString(std::move(s), state->rootPath(CanonPath::fromCwd()), staticEnv); + return state->parseExprFromString(std::move(s), state->rootPath("."), staticEnv); } @@ -915,7 +953,7 @@ std::unique_ptr AbstractNixRepl::create( } -void AbstractNixRepl::runSimple( +ReplExitStatus AbstractNixRepl::runSimple( ref evalState, const ValMap & extraEnv) { @@ -937,7 +975,7 @@ void AbstractNixRepl::runSimple( for (auto & [name, value] : extraEnv) repl->addVarToScope(repl->state->symbols.create(name), *value); - repl->mainLoop(); + return repl->mainLoop(); } } diff --git a/src/libcmd/repl.hh b/src/libcmd/repl.hh index 6d88883fe..21aa8bfc7 100644 --- a/src/libcmd/repl.hh +++ b/src/libcmd/repl.hh @@ -28,13 +28,13 @@ struct AbstractNixRepl const SearchPath & searchPath, nix::ref store, ref state, std::function getValues); - static void runSimple( + static ReplExitStatus runSimple( ref evalState, const ValMap & extraEnv); virtual void initEnv() = 0; - virtual void mainLoop() = 0; + virtual ReplExitStatus mainLoop() = 0; }; } diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 7481a2232..d6befd362 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -65,10 +65,10 @@ std::pair findAlongAttrPath(EvalState & state, const std::strin if (!attrIndex) { if (v->type() != nAttrs) - throw TypeError( + state.error( "the expression selected by the selection path '%1%' should be a set but is %2%", attrPath, - showType(*v)); + showType(*v)).debugThrow(); if (attr.empty()) throw Error("empty attribute name in selection path '%1%'", attrPath); @@ -88,10 +88,10 @@ std::pair findAlongAttrPath(EvalState & state, const std::strin else { if (!v->isList()) - throw TypeError( + state.error( "the expression selected by the selection path '%1%' should be a list but is %2%", attrPath, - showType(*v)); + showType(*v)).debugThrow(); if (*attrIndex >= v->listSize()) throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", *attrIndex, attrPath); diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 5808d58b6..2fc69e796 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -491,7 +491,7 @@ std::shared_ptr AttrCursor::maybeGetAttr(Symbol name, bool forceErro if (forceErrors) debug("reevaluating failed cached attribute '%s'", getAttrPathStr(name)); else - throw CachedEvalError("cached failure of attribute '%s'", getAttrPathStr(name)); + throw CachedEvalError(root->state, "cached failure of attribute '%s'", getAttrPathStr(name)); } else return std::make_shared(root, std::make_pair(shared_from_this(), name), nullptr, std::move(attr)); @@ -500,7 +500,7 @@ std::shared_ptr AttrCursor::maybeGetAttr(Symbol name, bool forceErro // evaluate to see whether 'name' exists } else return nullptr; - //throw TypeError("'%s' is not an attribute set", getAttrPathStr()); + //error("'%s' is not an attribute set", getAttrPathStr()).debugThrow(); } } @@ -508,7 +508,7 @@ std::shared_ptr AttrCursor::maybeGetAttr(Symbol name, bool forceErro if (v.type() != nAttrs) return nullptr; - //throw TypeError("'%s' is not an attribute set", getAttrPathStr()); + //error("'%s' is not an attribute set", getAttrPathStr()).debugThrow(); auto attr = v.attrs->get(name); @@ -574,14 +574,14 @@ std::string AttrCursor::getString() debug("using cached string attribute '%s'", getAttrPathStr()); return s->first; } else - root->state.error("'%s' is not a string", getAttrPathStr()).debugThrow(); + root->state.error("'%s' is not a string", getAttrPathStr()).debugThrow(); } } auto & v = forceValue(); if (v.type() != nString && v.type() != nPath) - root->state.error("'%s' is not a string but %s", getAttrPathStr()).debugThrow(); + root->state.error("'%s' is not a string but %s", getAttrPathStr()).debugThrow(); return v.type() == nString ? v.c_str() : v.path().to_string(); } @@ -616,7 +616,7 @@ string_t AttrCursor::getStringWithContext() return *s; } } else - root->state.error("'%s' is not a string", getAttrPathStr()).debugThrow(); + root->state.error("'%s' is not a string", getAttrPathStr()).debugThrow(); } } @@ -630,7 +630,7 @@ string_t AttrCursor::getStringWithContext() else if (v.type() == nPath) return {v.path().to_string(), {}}; else - root->state.error("'%s' is not a string but %s", getAttrPathStr()).debugThrow(); + root->state.error("'%s' is not a string but %s", getAttrPathStr()).debugThrow(); } bool AttrCursor::getBool() @@ -643,14 +643,14 @@ bool AttrCursor::getBool() debug("using cached Boolean attribute '%s'", getAttrPathStr()); return *b; } else - root->state.error("'%s' is not a Boolean", getAttrPathStr()).debugThrow(); + root->state.error("'%s' is not a Boolean", getAttrPathStr()).debugThrow(); } } auto & v = forceValue(); if (v.type() != nBool) - root->state.error("'%s' is not a Boolean", getAttrPathStr()).debugThrow(); + root->state.error("'%s' is not a Boolean", getAttrPathStr()).debugThrow(); return v.boolean; } @@ -665,14 +665,14 @@ NixInt AttrCursor::getInt() debug("using cached integer attribute '%s'", getAttrPathStr()); return i->x; } else - throw TypeError("'%s' is not an integer", getAttrPathStr()); + root->state.error("'%s' is not an integer", getAttrPathStr()).debugThrow(); } } auto & v = forceValue(); if (v.type() != nInt) - throw TypeError("'%s' is not an integer", getAttrPathStr()); + root->state.error("'%s' is not an integer", getAttrPathStr()).debugThrow(); return v.integer; } @@ -687,7 +687,7 @@ std::vector AttrCursor::getListOfStrings() debug("using cached list of strings attribute '%s'", getAttrPathStr()); return *l; } else - throw TypeError("'%s' is not a list of strings", getAttrPathStr()); + root->state.error("'%s' is not a list of strings", getAttrPathStr()).debugThrow(); } } @@ -697,7 +697,7 @@ std::vector AttrCursor::getListOfStrings() root->state.forceValue(v, noPos); if (v.type() != nList) - throw TypeError("'%s' is not a list", getAttrPathStr()); + root->state.error("'%s' is not a list", getAttrPathStr()).debugThrow(); std::vector res; @@ -720,14 +720,14 @@ std::vector AttrCursor::getAttrs() debug("using cached attrset attribute '%s'", getAttrPathStr()); return *attrs; } else - root->state.error("'%s' is not an attribute set", getAttrPathStr()).debugThrow(); + root->state.error("'%s' is not an attribute set", getAttrPathStr()).debugThrow(); } } auto & v = forceValue(); if (v.type() != nAttrs) - root->state.error("'%s' is not an attribute set", getAttrPathStr()).debugThrow(); + root->state.error("'%s' is not an attribute set", getAttrPathStr()).debugThrow(); std::vector attrs; for (auto & attr : *getValue().attrs) diff --git a/src/libexpr/eval-error.cc b/src/libexpr/eval-error.cc new file mode 100644 index 000000000..8db03610b --- /dev/null +++ b/src/libexpr/eval-error.cc @@ -0,0 +1,105 @@ +#include "eval-error.hh" +#include "eval.hh" +#include "value.hh" + +namespace nix { + +template +EvalErrorBuilder & EvalErrorBuilder::withExitStatus(unsigned int exitStatus) +{ + error.withExitStatus(exitStatus); + return *this; +} + +template +EvalErrorBuilder & EvalErrorBuilder::atPos(PosIdx pos) +{ + error.err.pos = error.state.positions[pos]; + return *this; +} + +template +EvalErrorBuilder & EvalErrorBuilder::atPos(Value & value, PosIdx fallback) +{ + return atPos(value.determinePos(fallback)); +} + +template +EvalErrorBuilder & EvalErrorBuilder::withTrace(PosIdx pos, const std::string_view text) +{ + error.err.traces.push_front( + Trace{.pos = error.state.positions[pos], .hint = HintFmt(std::string(text))}); + return *this; +} + +template +EvalErrorBuilder & EvalErrorBuilder::withSuggestions(Suggestions & s) +{ + error.err.suggestions = s; + return *this; +} + +template +EvalErrorBuilder & EvalErrorBuilder::withFrame(const Env & env, const Expr & expr) +{ + // NOTE: This is abusing side-effects. + // TODO: check compatibility with nested debugger calls. + // TODO: What side-effects?? + error.state.debugTraces.push_front(DebugTrace{ + .pos = error.state.positions[expr.getPos()], + .expr = expr, + .env = env, + .hint = HintFmt("Fake frame for debugging purposes"), + .isError = true}); + return *this; +} + +template +EvalErrorBuilder & EvalErrorBuilder::addTrace(PosIdx pos, HintFmt hint) +{ + error.addTrace(error.state.positions[pos], hint); + return *this; +} + +template +template +EvalErrorBuilder & +EvalErrorBuilder::addTrace(PosIdx pos, std::string_view formatString, const Args &... formatArgs) +{ + + addTrace(error.state.positions[pos], HintFmt(std::string(formatString), formatArgs...)); + return *this; +} + +template +void EvalErrorBuilder::debugThrow() +{ + if (error.state.debugRepl && !error.state.debugTraces.empty()) { + const DebugTrace & last = error.state.debugTraces.front(); + const Env * env = &last.env; + const Expr * expr = &last.expr; + error.state.runDebugRepl(&error, *env, *expr); + } + + // `EvalState` is the only class that can construct an `EvalErrorBuilder`, + // and it does so in dynamic storage. This is the final method called on + // any such instance and must delete itself before throwing the underlying + // error. + auto error = std::move(this->error); + delete this; + + throw error; +} + +template class EvalErrorBuilder; +template class EvalErrorBuilder; +template class EvalErrorBuilder; +template class EvalErrorBuilder; +template class EvalErrorBuilder; +template class EvalErrorBuilder; +template class EvalErrorBuilder; +template class EvalErrorBuilder; +template class EvalErrorBuilder; +template class EvalErrorBuilder; + +} diff --git a/src/libexpr/eval-error.hh b/src/libexpr/eval-error.hh new file mode 100644 index 000000000..7e0cbe982 --- /dev/null +++ b/src/libexpr/eval-error.hh @@ -0,0 +1,104 @@ +#pragma once + +#include + +#include "error.hh" +#include "pos-idx.hh" + +namespace nix { + +struct Env; +struct Expr; +struct Value; + +class EvalState; +template +class EvalErrorBuilder; + +class EvalError : public Error +{ + template + friend class EvalErrorBuilder; +public: + EvalState & state; + + EvalError(EvalState & state, ErrorInfo && errorInfo) + : Error(errorInfo) + , state(state) + { + } + + template + explicit EvalError(EvalState & state, const std::string & formatString, const Args &... formatArgs) + : Error(formatString, formatArgs...) + , state(state) + { + } +}; + +MakeError(ParseError, Error); +MakeError(AssertionError, EvalError); +MakeError(ThrownError, AssertionError); +MakeError(Abort, EvalError); +MakeError(TypeError, EvalError); +MakeError(UndefinedVarError, EvalError); +MakeError(MissingArgumentError, EvalError); +MakeError(CachedEvalError, EvalError); +MakeError(InfiniteRecursionError, EvalError); + +struct InvalidPathError : public EvalError +{ +public: + Path path; + InvalidPathError(EvalState & state, const Path & path) + : EvalError(state, "path '%s' is not valid", path) + { + } +}; + +/** + * `EvalErrorBuilder`s may only be constructed by `EvalState`. The `debugThrow` + * method must be the final method in any such `EvalErrorBuilder` usage, and it + * handles deleting the object. + */ +template +class EvalErrorBuilder final +{ + friend class EvalState; + + template + explicit EvalErrorBuilder(EvalState & state, const Args &... args) + : error(T(state, args...)) + { + } + +public: + T error; + + [[nodiscard, gnu::noinline]] EvalErrorBuilder & withExitStatus(unsigned int exitStatus); + + [[nodiscard, gnu::noinline]] EvalErrorBuilder & atPos(PosIdx pos); + + [[nodiscard, gnu::noinline]] EvalErrorBuilder & atPos(Value & value, PosIdx fallback = noPos); + + [[nodiscard, gnu::noinline]] EvalErrorBuilder & withTrace(PosIdx pos, const std::string_view text); + + [[nodiscard, gnu::noinline]] EvalErrorBuilder & withFrameTrace(PosIdx pos, const std::string_view text); + + [[nodiscard, gnu::noinline]] EvalErrorBuilder & withSuggestions(Suggestions & s); + + [[nodiscard, gnu::noinline]] EvalErrorBuilder & withFrame(const Env & e, const Expr & ex); + + [[nodiscard, gnu::noinline]] EvalErrorBuilder & addTrace(PosIdx pos, HintFmt hint); + + template + [[nodiscard, gnu::noinline]] EvalErrorBuilder & + addTrace(PosIdx pos, std::string_view formatString, const Args &... formatArgs); + + /** + * Delete the `EvalErrorBuilder` and throw the underlying exception. + */ + [[gnu::noinline, gnu::noreturn]] void debugThrow(); +}; + +} diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index 42cb68bbe..03320c7c9 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -3,6 +3,7 @@ #include "print.hh" #include "eval.hh" +#include "eval-error.hh" namespace nix { @@ -115,10 +116,11 @@ inline void EvalState::forceAttrs(Value & v, Callable getPos, std::string_view e PosIdx pos = getPos(); forceValue(v, pos); if (v.type() != nAttrs) { - error("expected a set but found %1%: %2%", - showType(v), - ValuePrinter(*this, v, errorPrintOptions)) - .withTrace(pos, errorCtx).debugThrow(); + error( + "expected a set but found %1%: %2%", + showType(v), + ValuePrinter(*this, v, errorPrintOptions) + ).withTrace(pos, errorCtx).debugThrow(); } } @@ -128,10 +130,11 @@ inline void EvalState::forceList(Value & v, const PosIdx pos, std::string_view e { forceValue(v, pos); if (!v.isList()) { - error("expected a list but found %1%: %2%", - showType(v), - ValuePrinter(*this, v, errorPrintOptions)) - .withTrace(pos, errorCtx).debugThrow(); + error( + "expected a list but found %1%: %2%", + showType(v), + ValuePrinter(*this, v, errorPrintOptions) + ).withTrace(pos, errorCtx).debugThrow(); } } diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index 2f6c12d45..b5783d28f 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -127,6 +127,16 @@ struct EvalSettings : Config Setting maxCallDepth{this, 10000, "max-call-depth", "The maximum function call depth to allow before erroring."}; + + Setting builtinsTraceDebugger{this, false, "debugger-on-trace", + R"( + If set to true and the `--debugger` flag is given, + [`builtins.trace`](@docroot@/language/builtins.md#builtins-trace) will + enter the debugger like + [`builtins.break`](@docroot@/language/builtins.md#builtins-break). + + This is useful for debugging warnings in third-party Nix code. + )"}; }; extern EvalSettings evalSettings; diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 91fd3ddf8..f2bbf20bb 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -3,6 +3,7 @@ #include "hash.hh" #include "primops.hh" #include "print-options.hh" +#include "shared.hh" #include "types.hh" #include "util.hh" #include "store-api.hh" @@ -339,46 +340,6 @@ void initGC() gcInitialised = true; } - -ErrorBuilder & ErrorBuilder::atPos(PosIdx pos) -{ - info.errPos = state.positions[pos]; - return *this; -} - -ErrorBuilder & ErrorBuilder::withTrace(PosIdx pos, const std::string_view text) -{ - info.traces.push_front(Trace{ .pos = state.positions[pos], .hint = hintformat(std::string(text)), .frame = false }); - return *this; -} - -ErrorBuilder & ErrorBuilder::withFrameTrace(PosIdx pos, const std::string_view text) -{ - info.traces.push_front(Trace{ .pos = state.positions[pos], .hint = hintformat(std::string(text)), .frame = true }); - return *this; -} - -ErrorBuilder & ErrorBuilder::withSuggestions(Suggestions & s) -{ - info.suggestions = s; - return *this; -} - -ErrorBuilder & ErrorBuilder::withFrame(const Env & env, const Expr & expr) -{ - // NOTE: This is abusing side-effects. - // TODO: check compatibility with nested debugger calls. - state.debugTraces.push_front(DebugTrace { - .pos = nullptr, - .expr = expr, - .env = env, - .hint = hintformat("Fake frame for debugging purposes"), - .isError = true - }); - return *this; -} - - EvalState::EvalState( const SearchPath & _searchPath, ref store, @@ -434,14 +395,14 @@ EvalState::EvalState( , emptyBindings(0) , rootFS( evalSettings.restrictEval || evalSettings.pureEval - ? ref(AllowListInputAccessor::create(makeFSInputAccessor(CanonPath::root), {}, + ? ref(AllowListInputAccessor::create(makeFSInputAccessor(), {}, [](const CanonPath & path) -> RestrictedPathError { auto modeInformation = evalSettings.pureEval ? "in pure evaluation mode (use '--impure' to override)" : "in restricted mode"; throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation); })) - : makeFSInputAccessor(CanonPath::root)) + : makeFSInputAccessor()) , corepkgsFS(makeMemoryInputAccessor()) , internalFS(makeMemoryInputAccessor()) , derivationInternal{corepkgsFS->addFile( @@ -456,7 +417,6 @@ EvalState::EvalState( , buildStore(buildStore ? buildStore : store) , debugRepl(nullptr) , debugStop(false) - , debugQuit(false) , trylevel(0) , regexCache(makeRegexCache()) #if HAVE_BOEHMGC @@ -507,13 +467,13 @@ EvalState::~EvalState() void EvalState::allowPath(const Path & path) { if (auto rootFS2 = rootFS.dynamic_pointer_cast()) - rootFS2->allowPath(CanonPath(path)); + rootFS2->allowPrefix(CanonPath(path)); } void EvalState::allowPath(const StorePath & storePath) { if (auto rootFS2 = rootFS.dynamic_pointer_cast()) - rootFS2->allowPath(CanonPath(store->toRealPath(storePath))); + rootFS2->allowPrefix(CanonPath(store->toRealPath(storePath))); } void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & v) @@ -744,7 +704,8 @@ void printEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env & if (se.up && env.up) { std::cout << "static: "; printStaticEnvBindings(st, se); - printWithBindings(st, env); + if (se.isWith) + printWithBindings(st, env); std::cout << std::endl; printEnvBindings(st, *se.up, *env.up, ++lvl); } else { @@ -756,7 +717,8 @@ void printEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env & std::cout << st[i.first] << " "; std::cout << ANSI_NORMAL; std::cout << std::endl; - printWithBindings(st, env); // probably nothing there for the top level. + if (se.isWith) + printWithBindings(st, env); // probably nothing there for the top level. std::cout << std::endl; } @@ -778,7 +740,7 @@ void mapStaticEnvBindings(const SymbolTable & st, const StaticEnv & se, const En if (env.up && se.up) { mapStaticEnvBindings(st, *se.up, *env.up, vm); - if (!env.values[0]->isThunk()) { + if (se.isWith && !env.values[0]->isThunk()) { // add 'with' bindings. Bindings::iterator j = env.values[0]->attrs->begin(); while (j != env.values[0]->attrs->end()) { @@ -811,7 +773,7 @@ void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr & ? std::make_unique( *this, DebugTrace { - .pos = error->info().errPos ? error->info().errPos : positions[expr.getPos()], + .pos = error->info().pos ? error->info().pos : positions[expr.getPos()], .expr = expr, .env = env, .hint = error->info().msg, @@ -821,45 +783,55 @@ void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr & if (error) { - printError("%s\n\n", error->what()); + printError("%s\n", error->what()); if (trylevel > 0 && error->info().level != lvlInfo) printError("This exception occurred in a 'tryEval' call. Use " ANSI_GREEN "--ignore-try" ANSI_NORMAL " to skip these.\n"); - - printError(ANSI_BOLD "Starting REPL to allow you to inspect the current state of the evaluator.\n" ANSI_NORMAL); } auto se = getStaticEnv(expr); if (se) { auto vm = mapStaticEnvBindings(symbols, *se.get(), env); - (debugRepl)(ref(shared_from_this()), *vm); + auto exitStatus = (debugRepl)(ref(shared_from_this()), *vm); + switch (exitStatus) { + case ReplExitStatus::QuitAll: + if (error) + throw *error; + throw Exit(0); + case ReplExitStatus::Continue: + break; + default: + abort(); + } } } -void EvalState::addErrorTrace(Error & e, const char * s, const std::string & s2) const +template +void EvalState::addErrorTrace(Error & e, const Args & ... formatArgs) const { - e.addTrace(nullptr, s, s2); + e.addTrace(nullptr, HintFmt(formatArgs...)); } -void EvalState::addErrorTrace(Error & e, const PosIdx pos, const char * s, const std::string & s2, bool frame) const +template +void EvalState::addErrorTrace(Error & e, const PosIdx pos, const Args & ... formatArgs) const { - e.addTrace(positions[pos], hintfmt(s, s2), frame); + e.addTrace(positions[pos], HintFmt(formatArgs...)); } +template static std::unique_ptr makeDebugTraceStacker( EvalState & state, Expr & expr, Env & env, std::shared_ptr && pos, - const char * s, - const std::string & s2) + const Args & ... formatArgs) { return std::make_unique(state, DebugTrace { .pos = std::move(pos), .expr = expr, .env = env, - .hint = hintfmt(s, s2), + .hint = HintFmt(formatArgs...), .isError = false }); } @@ -930,7 +902,7 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) return j->value; } if (!fromWith->parentWith) - error("undefined variable '%1%'", symbols[var.name]).atPos(var.pos).withFrame(*env, var).debugThrow(); + error("undefined variable '%1%'", symbols[var.name]).atPos(var.pos).withFrame(*env, var).debugThrow(); for (size_t l = fromWith->prevWith; l; --l, env = env->up) ; fromWith = fromWith->parentWith; } @@ -1136,7 +1108,7 @@ void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial) // computation. if (mustBeTrivial && !(dynamic_cast(e))) - error("file '%s' must be an attribute set", path).debugThrow(); + error("file '%s' must be an attribute set", path).debugThrow(); eval(e, v); } catch (Error & e) { addErrorTrace(e, "while evaluating the file '%1%':", resolvedPath.to_string()); @@ -1167,10 +1139,11 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const PosIdx pos, std::stri Value v; e->eval(*this, env, v); if (v.type() != nBool) - error("expected a Boolean but found %1%: %2%", - showType(v), - ValuePrinter(*this, v, errorPrintOptions)) - .withFrame(env, *e).debugThrow(); + error( + "expected a Boolean but found %1%: %2%", + showType(v), + ValuePrinter(*this, v, errorPrintOptions) + ).atPos(pos).withFrame(env, *e).debugThrow(); return v.boolean; } catch (Error & e) { e.addTrace(positions[pos], errorCtx); @@ -1184,10 +1157,11 @@ inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v, const PosIdx po try { e->eval(*this, env, v); if (v.type() != nAttrs) - error("expected a set but found %1%: %2%", - showType(v), - ValuePrinter(*this, v, errorPrintOptions)) - .withFrame(env, *e).debugThrow(); + error( + "expected a set but found %1%: %2%", + showType(v), + ValuePrinter(*this, v, errorPrintOptions) + ).withFrame(env, *e).debugThrow(); } catch (Error & e) { e.addTrace(positions[pos], errorCtx); throw; @@ -1224,6 +1198,18 @@ void ExprPath::eval(EvalState & state, Env & env, Value & v) } +Env * ExprAttrs::buildInheritFromEnv(EvalState & state, Env & up) +{ + Env & inheritEnv = state.allocEnv(inheritFromExprs->size()); + inheritEnv.up = &up; + + Displacement displ = 0; + for (auto from : *inheritFromExprs) + inheritEnv.values[displ++] = from->maybeThunk(state, up); + + return &inheritEnv; +} + void ExprAttrs::eval(EvalState & state, Env & env, Value & v) { v.mkAttrs(state.buildBindings(attrs.size() + dynamicAttrs.size()).finish()); @@ -1235,6 +1221,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) Env & env2(state.allocEnv(attrs.size())); env2.up = &env; dynamicEnv = &env2; + Env * inheritEnv = inheritFromExprs ? buildInheritFromEnv(state, env2) : nullptr; AttrDefs::iterator overrides = attrs.find(state.sOverrides); bool hasOverrides = overrides != attrs.end(); @@ -1245,11 +1232,11 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) Displacement displ = 0; for (auto & i : attrs) { Value * vAttr; - if (hasOverrides && !i.second.inherited) { + if (hasOverrides && i.second.kind != AttrDef::Kind::Inherited) { vAttr = state.allocValue(); - mkThunk(*vAttr, env2, i.second.e); + mkThunk(*vAttr, *i.second.chooseByKind(&env2, &env, inheritEnv), i.second.e); } else - vAttr = i.second.e->maybeThunk(state, i.second.inherited ? env : env2); + vAttr = i.second.e->maybeThunk(state, *i.second.chooseByKind(&env2, &env, inheritEnv)); env2.values[displ++] = vAttr; v.attrs->push_back(Attr(i.first, vAttr, i.second.pos)); } @@ -1281,9 +1268,15 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) } } - else - for (auto & i : attrs) - v.attrs->push_back(Attr(i.first, i.second.e->maybeThunk(state, env), i.second.pos)); + else { + Env * inheritEnv = inheritFromExprs ? buildInheritFromEnv(state, env) : nullptr; + for (auto & i : attrs) { + v.attrs->push_back(Attr( + i.first, + i.second.e->maybeThunk(state, *i.second.chooseByKind(&env, &env, inheritEnv)), + i.second.pos)); + } + } /* Dynamic attrs apply *after* rec and __overrides. */ for (auto & i : dynamicAttrs) { @@ -1296,7 +1289,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) auto nameSym = state.symbols.create(nameVal.string_view()); Bindings::iterator j = v.attrs->find(nameSym); if (j != v.attrs->end()) - state.error("dynamic attribute '%1%' already defined at %2%", state.symbols[nameSym], state.positions[j->pos]).atPos(i.pos).withFrame(env, *this).debugThrow(); + state.error("dynamic attribute '%1%' already defined at %2%", state.symbols[nameSym], state.positions[j->pos]).atPos(i.pos).withFrame(env, *this).debugThrow(); i.valueExpr->setName(nameSym); /* Keep sorted order so find can catch duplicates */ @@ -1315,12 +1308,30 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v) Env & env2(state.allocEnv(attrs->attrs.size())); env2.up = &env; + Env * inheritEnv = attrs->inheritFromExprs ? attrs->buildInheritFromEnv(state, env2) : nullptr; + /* The recursive attributes are evaluated in the new environment, while the inherited attributes are evaluated in the original environment. */ Displacement displ = 0; - for (auto & i : attrs->attrs) - env2.values[displ++] = i.second.e->maybeThunk(state, i.second.inherited ? env : env2); + for (auto & i : attrs->attrs) { + env2.values[displ++] = i.second.e->maybeThunk( + state, + *i.second.chooseByKind(&env2, &env, inheritEnv)); + } + + auto dts = state.debugRepl + ? makeDebugTraceStacker( + state, + *this, + env2, + getPos() + ? std::make_shared(state.positions[getPos()]) + : nullptr, + "while evaluating a '%1%' expression", + "let" + ) + : nullptr; body->eval(state, env2, v); } @@ -1384,7 +1395,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) state, *this, env, - state.positions[pos2], + state.positions[getPos()], "while evaluating the attribute '%1%'", showAttrPath(state, env, attrPath)) : nullptr; @@ -1408,8 +1419,8 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) for (auto & attr : *vAttrs->attrs) allAttrNames.insert(state.symbols[attr.name]); auto suggestions = Suggestions::bestMatches(allAttrNames, state.symbols[name]); - state.error("attribute '%1%' missing", state.symbols[name]) - .atPos(pos).withSuggestions(suggestions).withFrame(env, *this).debugThrow(); + state.error("attribute '%1%' missing", state.symbols[name]) + .atPos(pos).withSuggestions(suggestions).withFrame(env, *this).debugThrow(); } } vAttrs = j->value; @@ -1482,7 +1493,7 @@ public: void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const PosIdx pos) { if (callDepth > evalSettings.maxCallDepth) - error("stack overflow; max-call-depth exceeded").atPos(pos).template debugThrow(); + error("stack overflow; max-call-depth exceeded").atPos(pos).debugThrow(); CallDepth _level(callDepth); auto trace = evalSettings.traceFunctionCalls @@ -1540,13 +1551,13 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & auto j = args[0]->attrs->get(i.name); if (!j) { if (!i.def) { - error("function '%1%' called without required argument '%2%'", + error("function '%1%' called without required argument '%2%'", (lambda.name ? std::string(symbols[lambda.name]) : "anonymous lambda"), symbols[i.name]) .atPos(lambda.pos) .withTrace(pos, "from call site") .withFrame(*fun.lambda.env, lambda) - .debugThrow(); + .debugThrow(); } env2.values[displ++] = i.def->maybeThunk(*this, env2); } else { @@ -1566,14 +1577,14 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & for (auto & formal : lambda.formals->formals) formalNames.insert(symbols[formal.name]); auto suggestions = Suggestions::bestMatches(formalNames, symbols[i.name]); - error("function '%1%' called with unexpected argument '%2%'", + error("function '%1%' called with unexpected argument '%2%'", (lambda.name ? std::string(symbols[lambda.name]) : "anonymous lambda"), symbols[i.name]) .atPos(lambda.pos) .withTrace(pos, "from call site") .withSuggestions(suggestions) .withFrame(*fun.lambda.env, lambda) - .debugThrow(); + .debugThrow(); } abort(); // can't happen } @@ -1602,9 +1613,8 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & "while calling %s", lambda.name ? concatStrings("'", symbols[lambda.name], "'") - : "anonymous lambda", - true); - if (pos) addErrorTrace(e, pos, "from call site%s", "", true); + : "anonymous lambda"); + if (pos) addErrorTrace(e, pos, "from call site"); } throw; } @@ -1705,11 +1715,12 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & } else - error("attempt to call something which is not a function but %1%: %2%", + error( + "attempt to call something which is not a function but %1%: %2%", showType(vCur), ValuePrinter(*this, vCur, errorPrintOptions)) .atPos(pos) - .debugThrow(); + .debugThrow(); } vRes = vCur; @@ -1718,6 +1729,18 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & void ExprCall::eval(EvalState & state, Env & env, Value & v) { + auto dts = state.debugRepl + ? makeDebugTraceStacker( + state, + *this, + env, + getPos() + ? std::make_shared(state.positions[getPos()]) + : nullptr, + "while calling a function" + ) + : nullptr; + Value vFun; fun->eval(state, env, vFun); @@ -1779,12 +1802,12 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) if (j != args.end()) { attrs.insert(*j); } else if (!i.def) { - error(R"(cannot evaluate a function that has an argument without a value ('%1%') + error(R"(cannot evaluate a function that has an argument without a value ('%1%') Nix attempted to evaluate a function as a top level expression; in this case it must have its arguments supplied either by default values, or passed explicitly with '--arg' or '--argstr'. See https://nixos.org/manual/nix/stable/language/constructs.html#functions.)", symbols[i.name]) - .atPos(i.pos).withFrame(*fun.lambda.env, *fun.lambda.fun).debugThrow(); + .atPos(i.pos).withFrame(*fun.lambda.env, *fun.lambda.fun).debugThrow(); } } } @@ -1815,7 +1838,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v) if (!state.evalBool(env, cond, pos, "in the condition of the assert statement")) { std::ostringstream out; cond->show(state.symbols, out); - state.error("assertion '%1%' failed", out.str()).atPos(pos).withFrame(env, *this).debugThrow(); + state.error("assertion '%1%' failed", out.str()).atPos(pos).withFrame(env, *this).debugThrow(); } body->eval(state, env, v); } @@ -1993,14 +2016,14 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) nf = n; nf += vTmp.fpoint; } else - state.error("cannot add %1% to an integer", showType(vTmp)).atPos(i_pos).withFrame(env, *this).debugThrow(); + state.error("cannot add %1% to an integer", showType(vTmp)).atPos(i_pos).withFrame(env, *this).debugThrow(); } else if (firstType == nFloat) { if (vTmp.type() == nInt) { nf += vTmp.integer; } else if (vTmp.type() == nFloat) { nf += vTmp.fpoint; } else - state.error("cannot add %1% to a float", showType(vTmp)).atPos(i_pos).withFrame(env, *this).debugThrow(); + state.error("cannot add %1% to a float", showType(vTmp)).atPos(i_pos).withFrame(env, *this).debugThrow(); } else { if (s.empty()) s.reserve(es->size()); /* skip canonization of first path, which would only be not @@ -2022,7 +2045,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) v.mkFloat(nf); else if (firstType == nPath) { if (!context.empty()) - state.error("a string that refers to a store path cannot be appended to a path").atPos(pos).withFrame(env, *this).debugThrow(); + state.error("a string that refers to a store path cannot be appended to a path").atPos(pos).withFrame(env, *this).debugThrow(); v.mkPath(state.rootPath(CanonPath(canonPath(str())))); } else v.mkStringMove(c_str(), context); @@ -2037,8 +2060,9 @@ void ExprPos::eval(EvalState & state, Env & env, Value & v) void ExprBlackHole::eval(EvalState & state, Env & env, Value & v) { - state.error("infinite recursion encountered") - .debugThrow(); + state.error("infinite recursion encountered") + .atPos(v.determinePos(noPos)) + .debugThrow(); } // always force this to be separate, otherwise forceValue may inline it and take @@ -2052,7 +2076,7 @@ void EvalState::tryFixupBlackHolePos(Value & v, PosIdx pos) try { std::rethrow_exception(e); } catch (InfiniteRecursionError & e) { - e.err.errPos = positions[pos]; + e.atPos(positions[pos]); } catch (...) { } } @@ -2100,15 +2124,18 @@ NixInt EvalState::forceInt(Value & v, const PosIdx pos, std::string_view errorCt try { forceValue(v, pos); if (v.type() != nInt) - error("expected an integer but found %1%: %2%", - showType(v), - ValuePrinter(*this, v, errorPrintOptions)) - .debugThrow(); + error( + "expected an integer but found %1%: %2%", + showType(v), + ValuePrinter(*this, v, errorPrintOptions) + ).atPos(pos).debugThrow(); return v.integer; } catch (Error & e) { e.addTrace(positions[pos], errorCtx); throw; } + + return v.integer; } @@ -2119,10 +2146,11 @@ NixFloat EvalState::forceFloat(Value & v, const PosIdx pos, std::string_view err if (v.type() == nInt) return v.integer; else if (v.type() != nFloat) - error("expected a float but found %1%: %2%", - showType(v), - ValuePrinter(*this, v, errorPrintOptions)) - .debugThrow(); + error( + "expected a float but found %1%: %2%", + showType(v), + ValuePrinter(*this, v, errorPrintOptions) + ).atPos(pos).debugThrow(); return v.fpoint; } catch (Error & e) { e.addTrace(positions[pos], errorCtx); @@ -2136,15 +2164,18 @@ bool EvalState::forceBool(Value & v, const PosIdx pos, std::string_view errorCtx try { forceValue(v, pos); if (v.type() != nBool) - error("expected a Boolean but found %1%: %2%", - showType(v), - ValuePrinter(*this, v, errorPrintOptions)) - .debugThrow(); + error( + "expected a Boolean but found %1%: %2%", + showType(v), + ValuePrinter(*this, v, errorPrintOptions) + ).atPos(pos).debugThrow(); return v.boolean; } catch (Error & e) { e.addTrace(positions[pos], errorCtx); throw; } + + return v.boolean; } @@ -2159,10 +2190,11 @@ void EvalState::forceFunction(Value & v, const PosIdx pos, std::string_view erro try { forceValue(v, pos); if (v.type() != nFunction && !isFunctor(v)) - error("expected a function but found %1%: %2%", - showType(v), - ValuePrinter(*this, v, errorPrintOptions)) - .debugThrow(); + error( + "expected a function but found %1%: %2%", + showType(v), + ValuePrinter(*this, v, errorPrintOptions) + ).atPos(pos).debugThrow(); } catch (Error & e) { e.addTrace(positions[pos], errorCtx); throw; @@ -2175,10 +2207,11 @@ std::string_view EvalState::forceString(Value & v, const PosIdx pos, std::string try { forceValue(v, pos); if (v.type() != nString) - error("expected a string but found %1%: %2%", - showType(v), - ValuePrinter(*this, v, errorPrintOptions)) - .debugThrow(); + error( + "expected a string but found %1%: %2%", + showType(v), + ValuePrinter(*this, v, errorPrintOptions) + ).atPos(pos).debugThrow(); return v.string_view(); } catch (Error & e) { e.addTrace(positions[pos], errorCtx); @@ -2207,7 +2240,7 @@ std::string_view EvalState::forceStringNoCtx(Value & v, const PosIdx pos, std::s { auto s = forceString(v, pos, errorCtx); if (v.context()) { - error("the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string_view(), v.context()[0]).withTrace(pos, errorCtx).debugThrow(); + error("the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string_view(), v.context()[0]).withTrace(pos, errorCtx).debugThrow(); } return s; } @@ -2272,11 +2305,13 @@ BackedStringView EvalState::coerceToString( return std::move(*maybeString); auto i = v.attrs->find(sOutPath); if (i == v.attrs->end()) { - error("cannot coerce %1% to a string: %2%", - showType(v), - ValuePrinter(*this, v, errorPrintOptions)) + error( + "cannot coerce %1% to a string: %2%", + showType(v), + ValuePrinter(*this, v, errorPrintOptions) + ) .withTrace(pos, errorCtx) - .debugThrow(); + .debugThrow(); } return coerceToString(pos, *i->value, context, errorCtx, coerceMore, copyToStore, canonicalizePath); @@ -2284,7 +2319,7 @@ BackedStringView EvalState::coerceToString( if (v.type() == nExternal) { try { - return v.external->coerceToString(positions[pos], context, coerceMore, copyToStore); + return v.external->coerceToString(*this, pos, context, coerceMore, copyToStore); } catch (Error & e) { e.addTrace(nullptr, errorCtx); throw; @@ -2320,25 +2355,33 @@ BackedStringView EvalState::coerceToString( } } - error("cannot coerce %1% to a string: %2%", - showType(v), - ValuePrinter(*this, v, errorPrintOptions)) + error("cannot coerce %1% to a string: %2%", + showType(v), + ValuePrinter(*this, v, errorPrintOptions) + ) .withTrace(pos, errorCtx) - .debugThrow(); + .debugThrow(); } StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePath & path) { if (nix::isDerivation(path.path.abs())) - error("file names are not allowed to end in '%1%'", drvExtension).debugThrow(); + error("file names are not allowed to end in '%1%'", drvExtension).debugThrow(); auto i = srcToStore.find(path); auto dstPath = i != srcToStore.end() ? i->second : [&]() { - auto dstPath = fetchToStore(*store, path.resolveSymlinks(), path.baseName(), FileIngestionMethod::Recursive, nullptr, repair); + auto dstPath = fetchToStore( + *store, + path.resolveSymlinks(), + settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, + path.baseName(), + FileIngestionMethod::Recursive, + nullptr, + repair); allowPath(dstPath); srcToStore.insert_or_assign(path, dstPath); printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath)); @@ -2380,7 +2423,7 @@ SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext relative to the root filesystem. */ auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned(); if (path == "" || path[0] != '/') - error("string '%1%' doesn't represent an absolute path", path).withTrace(pos, errorCtx).debugThrow(); + error("string '%1%' doesn't represent an absolute path", path).withTrace(pos, errorCtx).debugThrow(); return rootPath(CanonPath(path)); } @@ -2390,7 +2433,7 @@ StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringCon auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned(); if (auto storePath = store->maybeParseStorePath(path)) return *storePath; - error("path '%1%' is not in the Nix store", path).withTrace(pos, errorCtx).debugThrow(); + error("path '%1%' is not in the Nix store", path).withTrace(pos, errorCtx).debugThrow(); } @@ -2400,18 +2443,18 @@ std::pair EvalState::coerceToSingleDerivedP auto s = forceString(v, context, pos, errorCtx); auto csize = context.size(); if (csize != 1) - error( + error( "string '%s' has %d entries in its context. It should only have exactly one entry", s, csize) - .withTrace(pos, errorCtx).debugThrow(); + .withTrace(pos, errorCtx).debugThrow(); auto derivedPath = std::visit(overloaded { [&](NixStringContextElem::Opaque && o) -> SingleDerivedPath { return std::move(o); }, [&](NixStringContextElem::DrvDeep &&) -> SingleDerivedPath { - error( + error( "string '%s' has a context which refers to a complete source and binary closure. This is not supported at this time", - s).withTrace(pos, errorCtx).debugThrow(); + s).withTrace(pos, errorCtx).debugThrow(); }, [&](NixStringContextElem::Built && b) -> SingleDerivedPath { return std::move(b); @@ -2434,16 +2477,16 @@ SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value & error message. */ std::visit(overloaded { [&](const SingleDerivedPath::Opaque & o) { - error( + error( "path string '%s' has context with the different path '%s'", s, sExpected) - .withTrace(pos, errorCtx).debugThrow(); + .withTrace(pos, errorCtx).debugThrow(); }, [&](const SingleDerivedPath::Built & b) { - error( + error( "string '%s' has context with the output '%s' from derivation '%s', but the string is not the right placeholder for this derivation output. It should be '%s'", s, b.output, b.drvPath->to_string(*store), sExpected) - .withTrace(pos, errorCtx).debugThrow(); + .withTrace(pos, errorCtx).debugThrow(); } }, derivedPath.raw()); } @@ -2528,7 +2571,7 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v case nThunk: // Must not be left by forceValue default: - error("cannot compare %1% with %2%", showType(v1), showType(v2)).withTrace(pos, errorCtx).debugThrow(); + error("cannot compare %1% with %2%", showType(v1), showType(v2)).withTrace(pos, errorCtx).debugThrow(); } } @@ -2689,14 +2732,14 @@ SourcePath resolveExprPath(SourcePath path) // Basic cycle/depth limit to avoid infinite loops. if (++followCount >= maxFollow) throw Error("too many symbolic links encountered while traversing the path '%s'", path); - auto p = path.parent().resolveSymlinks() + path.baseName(); + auto p = path.parent().resolveSymlinks() / path.baseName(); if (p.lstat().type != InputAccessor::tSymlink) break; path = {path.accessor, CanonPath(p.readLink(), path.path.parent().value_or(CanonPath::root))}; } /* If `path' refers to a directory, append `/default.nix'. */ if (path.resolveSymlinks().lstat().type == InputAccessor::tDirectory) - return path + "default.nix"; + return path / "default.nix"; return path; } @@ -2738,7 +2781,7 @@ Expr * EvalState::parseStdin() // drainFD should have left some extra space for terminators buffer.append("\0\0", 2); auto s = make_ref(std::move(buffer)); - return parse(s->data(), s->size(), Pos::Stdin{.source = s}, rootPath(CanonPath::fromCwd()), staticBaseEnv); + return parse(s->data(), s->size(), Pos::Stdin{.source = s}, rootPath("."), staticBaseEnv); } @@ -2767,13 +2810,12 @@ SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_ if (hasPrefix(path, "nix/")) return {corepkgsFS, CanonPath(path.substr(3))}; - debugThrow(ThrownError({ - .msg = hintfmt(evalSettings.pureEval + error( + evalSettings.pureEval ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)" : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)", - path), - .errPos = positions[pos] - }), 0, 0); + path + ).atPos(pos).debugThrow(); } @@ -2787,12 +2829,13 @@ std::optional EvalState::resolveSearchPathPath(const SearchPath::Pa if (EvalSettings::isPseudoUrl(value)) { try { - auto storePath = fetchers::downloadTarball( - store, EvalSettings::resolvePseudoUrl(value), "source", false).storePath; + auto accessor = fetchers::downloadTarball( + EvalSettings::resolvePseudoUrl(value)).accessor; + auto storePath = fetchToStore(*store, SourcePath(accessor), FetchMode::Copy); res = { store->toRealPath(storePath) }; - } catch (FileTransferError & e) { + } catch (Error & e) { logWarning({ - .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value) + .msg = HintFmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value) }); } } @@ -2825,7 +2868,7 @@ std::optional EvalState::resolveSearchPathPath(const SearchPath::Pa res = { path }; else { logWarning({ - .msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", value) + .msg = HintFmt("Nix search path entry '%1%' does not exist, ignoring", value) }); res = std::nullopt; } @@ -2856,11 +2899,11 @@ Expr * EvalState::parse( } -std::string ExternalValueBase::coerceToString(const Pos & pos, NixStringContext & context, bool copyMore, bool copyToStore) const +std::string ExternalValueBase::coerceToString(EvalState & state, const PosIdx & pos, NixStringContext & context, bool copyMore, bool copyToStore) const { - throw TypeError({ - .msg = hintfmt("cannot coerce %1% to a string: %2%", showType(), *this) - }); + state.error( + "cannot coerce %1% to a string: %2%", showType(), *this + ).atPos(pos).debugThrow(); } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 2368187b1..01abd4eb1 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -2,6 +2,7 @@ ///@file #include "attr-set.hh" +#include "eval-error.hh" #include "types.hh" #include "value.hh" #include "nixexpr.hh" @@ -10,6 +11,7 @@ #include "experimental-features.hh" #include "input-accessor.hh" #include "search-path.hh" +#include "repl-exit-status.hh" #include #include @@ -147,49 +149,10 @@ struct DebugTrace { std::shared_ptr pos; const Expr & expr; const Env & env; - hintformat hint; + HintFmt hint; bool isError; }; -void debugError(Error * e, Env & env, Expr & expr); - -class ErrorBuilder -{ - private: - EvalState & state; - ErrorInfo info; - - ErrorBuilder(EvalState & s, ErrorInfo && i): state(s), info(i) { } - - public: - template - [[nodiscard, gnu::noinline]] - static ErrorBuilder * create(EvalState & s, const Args & ... args) - { - return new ErrorBuilder(s, ErrorInfo { .msg = hintfmt(args...) }); - } - - [[nodiscard, gnu::noinline]] - ErrorBuilder & atPos(PosIdx pos); - - [[nodiscard, gnu::noinline]] - ErrorBuilder & withTrace(PosIdx pos, const std::string_view text); - - [[nodiscard, gnu::noinline]] - ErrorBuilder & withFrameTrace(PosIdx pos, const std::string_view text); - - [[nodiscard, gnu::noinline]] - ErrorBuilder & withSuggestions(Suggestions & s); - - [[nodiscard, gnu::noinline]] - ErrorBuilder & withFrame(const Env & e, const Expr & ex); - - template - [[gnu::noinline, gnu::noreturn]] - void debugThrow(); -}; - - class EvalState : public std::enable_shared_from_this { public: @@ -257,9 +220,8 @@ public: /** * Debugger */ - void (* debugRepl)(ref es, const ValMap & extraEnv); + ReplExitStatus (* debugRepl)(ref es, const ValMap & extraEnv); bool debugStop; - bool debugQuit; int trylevel; std::list debugTraces; std::map> exprEnvs; @@ -274,39 +236,11 @@ public: void runDebugRepl(const Error * error, const Env & env, const Expr & expr); - template - [[gnu::noinline, gnu::noreturn]] - void debugThrowLastTrace(E && error) - { - debugThrow(error, nullptr, nullptr); - } - - template - [[gnu::noinline, gnu::noreturn]] - void debugThrow(E && error, const Env * env, const Expr * expr) - { - if (debugRepl && ((env && expr) || !debugTraces.empty())) { - if (!env || !expr) { - const DebugTrace & last = debugTraces.front(); - env = &last.env; - expr = &last.expr; - } - runDebugRepl(&error, *env, *expr); - } - - throw std::move(error); - } - - // This is dangerous, but gets in line with the idea that error creation and - // throwing should not allocate on the stack of hot functions. - // as long as errors are immediately thrown, it works. - ErrorBuilder * errorBuilder; - - template + template [[nodiscard, gnu::noinline]] - ErrorBuilder & error(const Args & ... args) { - errorBuilder = ErrorBuilder::create(*this, args...); - return *errorBuilder; + EvalErrorBuilder & error(const Args & ... args) { + // `EvalErrorBuilder::debugThrow` performs the corresponding `delete`. + return *new EvalErrorBuilder(*this, args...); } private: @@ -372,6 +306,11 @@ public: */ SourcePath rootPath(CanonPath path); + /** + * Variant which accepts relative paths too. + */ + SourcePath rootPath(PathView path); + /** * Allow access to a path. */ @@ -493,10 +432,12 @@ public: std::string_view forceString(Value & v, NixStringContext & context, const PosIdx pos, std::string_view errorCtx); std::string_view forceStringNoCtx(Value & v, const PosIdx pos, std::string_view errorCtx); + template [[gnu::noinline]] - void addErrorTrace(Error & e, const char * s, const std::string & s2) const; + void addErrorTrace(Error & e, const Args & ... formatArgs) const; + template [[gnu::noinline]] - void addErrorTrace(Error & e, const PosIdx pos, const char * s, const std::string & s2, bool frame = false) const; + void addErrorTrace(Error & e, const PosIdx pos, const Args & ... formatArgs) const; public: /** @@ -819,7 +760,6 @@ struct DebugTraceStacker { DebugTraceStacker(EvalState & evalState, DebugTrace t); ~DebugTraceStacker() { - // assert(evalState.debugTraces.front() == trace); evalState.debugTraces.pop_front(); } EvalState & evalState; @@ -845,22 +785,6 @@ SourcePath resolveExprPath(SourcePath path); */ bool isAllowedURI(std::string_view uri, const Strings & allowedPaths); -struct InvalidPathError : EvalError -{ - Path path; - InvalidPathError(const Path & path); -#ifdef EXCEPTION_NEEDS_THROW_SPEC - ~InvalidPathError() throw () { }; -#endif -}; - -template -void ErrorBuilder::debugThrow() -{ - // NOTE: We always use the -LastTrace version as we push the new trace in withFrame() - state.debugThrowLastTrace(ErrorType(info)); -} - } #include "eval-inline.hh" diff --git a/src/libexpr/flake/call-flake.nix b/src/libexpr/flake/call-flake.nix index 4beb0b0fe..d0ccb1e37 100644 --- a/src/libexpr/flake/call-flake.nix +++ b/src/libexpr/flake/call-flake.nix @@ -1,20 +1,52 @@ -lockFileStr: rootSrc: rootSubdir: +# This is a helper to callFlake() to lazily fetch flake inputs. + +# The contents of the lock file, in JSON format. +lockFileStr: + +# A mapping of lock file node IDs to { sourceInfo, subdir } attrsets, +# with sourceInfo.outPath providing an InputAccessor to a previously +# fetched tree. This is necessary for possibly unlocked inputs, in +# particular the root input, but also --override-inputs pointing to +# unlocked trees. +overrides: let lockFile = builtins.fromJSON lockFileStr; + # Resolve a input spec into a node name. An input spec is + # either a node name, or a 'follows' path from the root + # node. + resolveInput = inputSpec: + if builtins.isList inputSpec + then getInputByPath lockFile.root inputSpec + else inputSpec; + + # Follow an input path (e.g. ["dwarffs" "nixpkgs"]) from the + # root node, returning the final node. + getInputByPath = nodeName: path: + if path == [] + then nodeName + else + getInputByPath + # Since this could be a 'follows' input, call resolveInput. + (resolveInput lockFile.nodes.${nodeName}.inputs.${builtins.head path}) + (builtins.tail path); + allNodes = builtins.mapAttrs (key: node: let sourceInfo = - if key == lockFile.root - then rootSrc - else fetchTree (node.info or {} // removeAttrs node.locked ["dir"]); + if overrides ? ${key} + then + overrides.${key}.sourceInfo + else + # FIXME: remove obsolete node.info. + fetchTree (node.info or {} // removeAttrs node.locked ["dir"]); - subdir = if key == lockFile.root then rootSubdir else node.locked.dir or ""; + subdir = overrides.${key}.dir or node.locked.dir or ""; outPath = sourceInfo + ((if subdir == "" then "" else "/") + subdir); @@ -24,25 +56,6 @@ let (inputName: inputSpec: allNodes.${resolveInput inputSpec}) (node.inputs or {}); - # Resolve a input spec into a node name. An input spec is - # either a node name, or a 'follows' path from the root - # node. - resolveInput = inputSpec: - if builtins.isList inputSpec - then getInputByPath lockFile.root inputSpec - else inputSpec; - - # Follow an input path (e.g. ["dwarffs" "nixpkgs"]) from the - # root node, returning the final node. - getInputByPath = nodeName: path: - if path == [] - then nodeName - else - getInputByPath - # Since this could be a 'follows' input, call resolveInput. - (resolveInput lockFile.nodes.${nodeName}.inputs.${builtins.head path}) - (builtins.tail path); - outputs = flake.outputs (inputs // { self = result; }); result = diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index fee58792b..022d39cdb 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -147,15 +147,15 @@ static FlakeInput parseFlakeInput(EvalState & state, NixStringContext emptyContext = {}; attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, emptyContext).dump()); } else - throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected", - state.symbols[attr.name], showType(*attr.value)); + state.error("flake input attribute '%s' is %s while a string, Boolean, or integer is expected", + state.symbols[attr.name], showType(*attr.value)).debugThrow(); } #pragma GCC diagnostic pop } } catch (Error & e) { e.addTrace( state.positions[attr.pos], - hintfmt("while evaluating flake attribute '%s'", state.symbols[attr.name])); + HintFmt("while evaluating flake attribute '%s'", state.symbols[attr.name])); throw; } } @@ -164,7 +164,7 @@ static FlakeInput parseFlakeInput(EvalState & state, try { input.ref = FlakeRef::fromAttrs(attrs); } catch (Error & e) { - e.addTrace(state.positions[pos], hintfmt("while evaluating flake input")); + e.addTrace(state.positions[pos], HintFmt("while evaluating flake input")); throw; } else { @@ -295,15 +295,15 @@ static Flake getFlake( std::vector ss; for (auto elem : setting.value->listItems()) { if (elem->type() != nString) - throw TypeError("list element in flake configuration setting '%s' is %s while a string is expected", - state.symbols[setting.name], showType(*setting.value)); + state.error("list element in flake configuration setting '%s' is %s while a string is expected", + state.symbols[setting.name], showType(*setting.value)).debugThrow(); ss.emplace_back(state.forceStringNoCtx(*elem, setting.pos, "")); } flake.config.settings.emplace(state.symbols[setting.name], ss); } else - throw TypeError("flake configuration setting '%s' is %s", - state.symbols[setting.name], showType(*setting.value)); + state.error("flake configuration setting '%s' is %s", + state.symbols[setting.name], showType(*setting.value)).debugThrow(); } } @@ -365,6 +365,7 @@ LockedFlake lockFlake( std::map overrides; std::set explicitCliOverrides; std::set overridesUsed, updatesUsed; + std::map, StorePath> nodePaths; for (auto & i : lockFlags.inputOverrides) { overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second }); @@ -535,11 +536,13 @@ LockedFlake lockFlake( } } - computeLocks( - mustRefetch - ? getFlake(state, oldLock->lockedRef, false, flakeCache, inputPath).inputs - : fakeInputs, - childNode, inputPath, oldLock, lockRootPath, parentPath, !mustRefetch); + if (mustRefetch) { + auto inputFlake = getFlake(state, oldLock->lockedRef, false, flakeCache, inputPath); + nodePaths.emplace(childNode, inputFlake.storePath); + computeLocks(inputFlake.inputs, childNode, inputPath, oldLock, lockRootPath, parentPath, false); + } else { + computeLocks(fakeInputs, childNode, inputPath, oldLock, lockRootPath, parentPath, true); + } } else { /* We need to create a new lock file entry. So fetch @@ -584,6 +587,7 @@ LockedFlake lockFlake( flake. Also, unless we already have this flake in the top-level lock file, use this flake's own lock file. */ + nodePaths.emplace(childNode, inputFlake.storePath); computeLocks( inputFlake.inputs, childNode, inputPath, oldLock @@ -596,11 +600,13 @@ LockedFlake lockFlake( } else { - auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree( + auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree( state, *input.ref, useRegistries, flakeCache); auto childNode = make_ref(lockedRef, ref, false); + nodePaths.emplace(childNode, storePath); + node->inputs.insert_or_assign(id, childNode); } } @@ -615,6 +621,8 @@ LockedFlake lockFlake( // Bring in the current ref for relative path resolution if we have it auto parentPath = canonPath(state.store->toRealPath(flake.storePath) + "/" + flake.lockedRef.subdir, true); + nodePaths.emplace(newLockFile.root, flake.storePath); + computeLocks( flake.inputs, newLockFile.root, @@ -707,14 +715,6 @@ LockedFlake lockFlake( flake.lockedRef.input.getRev() && prevLockedRef.input.getRev() != flake.lockedRef.input.getRev()) warn("committed new revision '%s'", flake.lockedRef.input.getRev()->gitRev()); - - /* Make sure that we picked up the change, - i.e. the tree should usually be dirty - now. Corner case: we could have reverted from a - dirty to a clean tree! */ - if (flake.lockedRef.input == prevLockedRef.input - && !flake.lockedRef.input.isLocked()) - throw Error("'%s' did not change after I updated its 'flake.lock' file; is 'flake.lock' under version control?", flake.originalRef); } } else throw Error("cannot write modified lock file of flake '%s' (use '--no-write-lock-file' to ignore)", topRef); @@ -724,7 +724,11 @@ LockedFlake lockFlake( } } - return LockedFlake { .flake = std::move(flake), .lockFile = std::move(newLockFile) }; + return LockedFlake { + .flake = std::move(flake), + .lockFile = std::move(newLockFile), + .nodePaths = std::move(nodePaths) + }; } catch (Error & e) { e.addTrace({}, "while updating the lock file of flake '%s'", flake.lockedRef.to_string()); @@ -736,30 +740,48 @@ void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & vRes) { - auto vLocks = state.allocValue(); - auto vRootSrc = state.allocValue(); - auto vRootSubdir = state.allocValue(); - auto vTmp1 = state.allocValue(); - auto vTmp2 = state.allocValue(); + experimentalFeatureSettings.require(Xp::Flakes); - vLocks->mkString(lockedFlake.lockFile.to_string()); + auto [lockFileStr, keyMap] = lockedFlake.lockFile.to_string(); - emitTreeAttrs( - state, - lockedFlake.flake.storePath, - lockedFlake.flake.lockedRef.input, - *vRootSrc, - false, - lockedFlake.flake.forceDirty); + auto overrides = state.buildBindings(lockedFlake.nodePaths.size()); - vRootSubdir->mkString(lockedFlake.flake.lockedRef.subdir); + for (auto & [node, storePath] : lockedFlake.nodePaths) { + auto override = state.buildBindings(2); + + auto & vSourceInfo = override.alloc(state.symbols.create("sourceInfo")); + + auto lockedNode = node.dynamic_pointer_cast(); + + emitTreeAttrs( + state, + storePath, + lockedNode ? lockedNode->lockedRef.input : lockedFlake.flake.lockedRef.input, + vSourceInfo, + false, + !lockedNode && lockedFlake.flake.forceDirty); + + auto key = keyMap.find(node); + assert(key != keyMap.end()); + + override + .alloc(state.symbols.create("dir")) + .mkString(lockedNode ? lockedNode->lockedRef.subdir : lockedFlake.flake.lockedRef.subdir); + + overrides.alloc(state.symbols.create(key->second)).mkAttrs(override); + } + + auto & vOverrides = state.allocValue()->mkAttrs(overrides); auto vCallFlake = state.allocValue(); state.evalFile(state.callFlakeInternal, *vCallFlake); + auto vTmp1 = state.allocValue(); + auto vLocks = state.allocValue(); + vLocks->mkString(lockFileStr); state.callFunction(*vCallFlake, *vLocks, *vTmp1, noPos); - state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos); - state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos); + + state.callFunction(*vTmp1, vOverrides, vRes, noPos); } static void prim_getFlake(EvalState & state, const PosIdx pos, Value * * args, Value & v) @@ -865,11 +887,11 @@ static void prim_flakeRefToString( attrs.emplace(state.symbols[attr.name], std::string(attr.value->string_view())); } else { - state.error( + state.error( "flake reference attribute sets may only contain integers, Booleans, " "and strings, but attribute '%s' is %s", state.symbols[attr.name], - showType(*attr.value)).debugThrow(); + showType(*attr.value)).debugThrow(); } } auto flakeRef = FlakeRef::fromAttrs(attrs); diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index d5ad3eade..19b680c56 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -103,6 +103,13 @@ struct LockedFlake Flake flake; LockFile lockFile; + /** + * Store paths of nodes that have been fetched in + * lockFlake(); in particular, the root node and the overriden + * inputs. + */ + std::map, StorePath> nodePaths; + Fingerprint getFingerprint() const; }; diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index 3e99fb2d4..e3a28c7c6 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -38,7 +38,7 @@ LockedNode::LockedNode(const nlohmann::json & json) , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true) { if (!lockedRef.input.isLocked()) - throw Error("lock file contains mutable lock '%s'", + throw Error("lock file contains unlocked input '%s'", fetchers::attrsToJSON(lockedRef.input.toAttrs())); } @@ -107,7 +107,7 @@ LockFile::LockFile(const nlohmann::json & json, const Path & path) std::string inputKey = i.value(); auto k = nodeMap.find(inputKey); if (k == nodeMap.end()) { - auto nodes = json["nodes"]; + auto & nodes = json["nodes"]; auto jsonNode2 = nodes.find(inputKey); if (jsonNode2 == nodes.end()) throw Error("lock file references missing node '%s'", inputKey); @@ -134,10 +134,10 @@ LockFile::LockFile(const nlohmann::json & json, const Path & path) // a bit since we don't need to worry about cycles. } -nlohmann::json LockFile::toJSON() const +std::pair LockFile::toJSON() const { nlohmann::json nodes; - std::unordered_map, std::string> nodeKeys; + KeyMap nodeKeys; std::unordered_set keys; std::function node)> dumpNode; @@ -194,12 +194,13 @@ nlohmann::json LockFile::toJSON() const json["root"] = dumpNode("root", root); json["nodes"] = std::move(nodes); - return json; + return {json, std::move(nodeKeys)}; } -std::string LockFile::to_string() const +std::pair LockFile::to_string() const { - return toJSON().dump(2); + auto [json, nodeKeys] = toJSON(); + return {json.dump(2), std::move(nodeKeys)}; } LockFile LockFile::read(const Path & path) @@ -210,7 +211,7 @@ LockFile LockFile::read(const Path & path) std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile) { - stream << lockFile.toJSON().dump(2); + stream << lockFile.toJSON().first.dump(2); return stream; } @@ -243,7 +244,7 @@ std::optional LockFile::isUnlocked() const bool LockFile::operator ==(const LockFile & other) const { // FIXME: slow - return toJSON() == other.toJSON(); + return toJSON().first == other.toJSON().first; } bool LockFile::operator !=(const LockFile & other) const diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh index 5a1493404..57a7202a2 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libexpr/flake/lockfile.hh @@ -59,14 +59,15 @@ struct LockFile typedef std::map, std::string> KeyMap; - nlohmann::json toJSON() const; + std::pair toJSON() const; - std::string to_string() const; + std::pair to_string() const; static LockFile read(const Path & path); /** - * Check whether this lock file has any unlocked inputs. + * Check whether this lock file has any unlocked inputs. If so, + * return one. */ std::optional isUnlocked() const; diff --git a/src/libexpr/flake/url-name.cc b/src/libexpr/flake/url-name.cc index 753f197d5..d62b34552 100644 --- a/src/libexpr/flake/url-name.cc +++ b/src/libexpr/flake/url-name.cc @@ -5,13 +5,12 @@ namespace nix { static const std::string attributeNamePattern("[a-zA-Z0-9_-]+"); -static const std::regex lastAttributeRegex("(?:" + attributeNamePattern + "\\.)*(?!default)(" + attributeNamePattern +")(\\^.*)?"); +static const std::regex lastAttributeRegex("^((?:" + attributeNamePattern + "\\.)*)(" + attributeNamePattern +")(\\^.*)?$"); static const std::string pathSegmentPattern("[a-zA-Z0-9_-]+"); static const std::regex lastPathSegmentRegex(".*/(" + pathSegmentPattern +")"); static const std::regex secondPathSegmentRegex("(?:" + pathSegmentPattern + ")/(" + pathSegmentPattern +")(?:/.*)?"); static const std::regex gitProviderRegex("github|gitlab|sourcehut"); static const std::regex gitSchemeRegex("git($|\\+.*)"); -static const std::regex defaultOutputRegex(".*\\.default($|\\^.*)"); std::optional getNameFromURL(const ParsedURL & url) { @@ -22,8 +21,11 @@ std::optional getNameFromURL(const ParsedURL & url) return url.query.at("dir"); /* If the fragment isn't a "default" and contains two attribute elements, use the last one */ - if (std::regex_match(url.fragment, match, lastAttributeRegex)) - return match.str(1); + if (std::regex_match(url.fragment, match, lastAttributeRegex) + && match.str(1) != "defaultPackage." + && match.str(2) != "default") { + return match.str(2); + } /* If this is a github/gitlab/sourcehut flake, use the repo name */ if (std::regex_match(url.scheme, gitProviderRegex) && std::regex_match(url.path, match, secondPathSegmentRegex)) @@ -33,10 +35,6 @@ std::optional getNameFromURL(const ParsedURL & url) if (std::regex_match(url.scheme, gitSchemeRegex) && std::regex_match(url.path, match, lastPathSegmentRegex)) return match.str(1); - /* If everything failed but there is a non-default fragment, use it in full */ - if (!url.fragment.empty() && !std::regex_match(url.fragment, defaultOutputRegex)) - return url.fragment; - /* If there is no fragment, take the last element of the path */ if (std::regex_match(url.path, match, lastPathSegmentRegex)) return match.str(1); diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 51449ccb3..e9ed1ef08 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -49,7 +49,7 @@ std::string PackageInfo::queryName() const { if (name == "" && attrs) { auto i = attrs->find(state->sName); - if (i == attrs->end()) throw TypeError("derivation name missing"); + if (i == attrs->end()) state->error("derivation name missing").debugThrow(); name = state->forceStringNoCtx(*i->value, noPos, "while evaluating the 'name' attribute of a derivation"); } return name; @@ -396,7 +396,8 @@ static void getDerivations(EvalState & state, Value & vIn, } } - else throw TypeError("expression does not evaluate to a derivation (or a set or list of those)"); + else + state.error("expression does not evaluate to a derivation (or a set or list of those)").debugThrow(); } diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 99a475ff9..2d12c47c5 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -1,4 +1,6 @@ #include "json-to-value.hh" +#include "value.hh" +#include "eval.hh" #include #include @@ -159,7 +161,7 @@ public: } bool parse_error(std::size_t, const std::string&, const nlohmann::detail::exception& ex) { - throw JSONParseError(ex.what()); + throw JSONParseError("%s", ex.what()); } }; diff --git a/src/libexpr/json-to-value.hh b/src/libexpr/json-to-value.hh index 3b8ec000f..3c8fa5cc0 100644 --- a/src/libexpr/json-to-value.hh +++ b/src/libexpr/json-to-value.hh @@ -1,13 +1,16 @@ #pragma once ///@file -#include "eval.hh" +#include "error.hh" #include namespace nix { -MakeError(JSONParseError, EvalError); +class EvalState; +struct Value; + +MakeError(JSONParseError, Error); void parseJSON(EvalState & state, const std::string_view & s, Value & v); diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index d7a0b5048..5b26d6927 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -94,6 +94,9 @@ static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length) } +// yacc generates code that uses unannotated fallthrough. +#pragma GCC diagnostic ignored "-Wimplicit-fallthrough" + #define YY_USER_INIT initLoc(yylloc) #define YY_USER_ACTION adjustLoc(yylloc, yytext, yyleng); @@ -146,9 +149,9 @@ or { return OR_KW; } try { yylval->n = boost::lexical_cast(yytext); } catch (const boost::bad_lexical_cast &) { - throw ParseError({ - .msg = hintfmt("invalid integer '%1%'", yytext), - .errPos = state->positions[CUR_POS], + throw ParseError(ErrorInfo{ + .msg = HintFmt("invalid integer '%1%'", yytext), + .pos = state->positions[CUR_POS], }); } return INT_LIT; @@ -156,9 +159,9 @@ or { return OR_KW; } {FLOAT} { errno = 0; yylval->nf = strtod(yytext, 0); if (errno != 0) - throw ParseError({ - .msg = hintfmt("invalid float '%1%'", yytext), - .errPos = state->positions[CUR_POS], + throw ParseError(ErrorInfo{ + .msg = HintFmt("invalid float '%1%'", yytext), + .pos = state->positions[CUR_POS], }); return FLOAT_LIT; } @@ -285,9 +288,9 @@ or { return OR_KW; } {ANY} | <> { - throw ParseError({ - .msg = hintfmt("path has a trailing slash"), - .errPos = state->positions[CUR_POS], + throw ParseError(ErrorInfo{ + .msg = HintFmt("path has a trailing slash"), + .pos = state->positions[CUR_POS], }); } diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 6fe4ba81b..4b805d710 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -70,10 +70,8 @@ void ExprOpHasAttr::show(const SymbolTable & symbols, std::ostream & str) const str << ") ? " << showAttrPath(symbols, attrPath) << ")"; } -void ExprAttrs::show(const SymbolTable & symbols, std::ostream & str) const +void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) const { - if (recursive) str << "rec "; - str << "{ "; typedef const decltype(attrs)::value_type * Attr; std::vector sorted; for (auto & i : attrs) sorted.push_back(&i); @@ -81,10 +79,37 @@ void ExprAttrs::show(const SymbolTable & symbols, std::ostream & str) const std::string_view sa = symbols[a->first], sb = symbols[b->first]; return sa < sb; }); + std::vector inherits; + std::map> inheritsFrom; for (auto & i : sorted) { - if (i->second.inherited) - str << "inherit " << symbols[i->first] << " " << "; "; - else { + switch (i->second.kind) { + case AttrDef::Kind::Plain: + break; + case AttrDef::Kind::Inherited: + inherits.push_back(i->first); + break; + case AttrDef::Kind::InheritedFrom: { + auto & select = dynamic_cast(*i->second.e); + auto & from = dynamic_cast(*select.e); + inheritsFrom[&from].push_back(i->first); + break; + } + } + } + if (!inherits.empty()) { + str << "inherit"; + for (auto sym : inherits) str << " " << symbols[sym]; + str << "; "; + } + for (const auto & [from, syms] : inheritsFrom) { + str << "inherit ("; + (*inheritFromExprs)[from->displ]->show(symbols, str); + str << ")"; + for (auto sym : syms) str << " " << symbols[sym]; + str << "; "; + } + for (auto & i : sorted) { + if (i->second.kind == AttrDef::Kind::Plain) { str << symbols[i->first] << " = "; i->second.e->show(symbols, str); str << "; "; @@ -97,6 +122,13 @@ void ExprAttrs::show(const SymbolTable & symbols, std::ostream & str) const i.valueExpr->show(symbols, str); str << "; "; } +} + +void ExprAttrs::show(const SymbolTable & symbols, std::ostream & str) const +{ + if (recursive) str << "rec "; + str << "{ "; + showBindings(symbols, str); str << "}"; } @@ -152,15 +184,7 @@ void ExprCall::show(const SymbolTable & symbols, std::ostream & str) const void ExprLet::show(const SymbolTable & symbols, std::ostream & str) const { str << "(let "; - for (auto & i : attrs->attrs) - if (i.second.inherited) { - str << "inherit " << symbols[i.first] << "; "; - } - else { - str << symbols[i.first] << " = "; - i.second.e->show(symbols, str); - str << "; "; - } + attrs->showBindings(symbols, str); str << "in "; body->show(symbols, str); str << ")"; @@ -296,15 +320,21 @@ void ExprVar::bindVars(EvalState & es, const std::shared_ptr & enclosing `with'. If there is no `with', then we can issue an "undefined variable" error now. */ if (withLevel == -1) - throw UndefinedVarError({ - .msg = hintfmt("undefined variable '%1%'", es.symbols[name]), - .errPos = es.positions[pos] - }); + es.error( + "undefined variable '%1%'", + es.symbols[name] + ).atPos(pos).debugThrow(); for (auto * e = env.get(); e && !fromWith; e = e->up) fromWith = e->isWith; this->level = withLevel; } +void ExprInheritFrom::bindVars(EvalState & es, const std::shared_ptr & env) +{ + if (es.debugRepl) + es.exprEnvs.insert(std::make_pair(this, env)); +} + void ExprSelect::bindVars(EvalState & es, const std::shared_ptr & env) { if (es.debugRepl) @@ -328,22 +358,47 @@ void ExprOpHasAttr::bindVars(EvalState & es, const std::shared_ptrbindVars(es, env); } +std::shared_ptr ExprAttrs::bindInheritSources( + EvalState & es, const std::shared_ptr & env) +{ + if (!inheritFromExprs) + return nullptr; + + // the inherit (from) source values are inserted into an env of its own, which + // does not introduce any variable names. + // analysis must see an empty env, or an env that contains only entries with + // otherwise unused names to not interfere with regular names. the parser + // has already filled all exprs that access this env with appropriate level + // and displacement, and nothing else is allowed to access it. ideally we'd + // not even *have* an expr that grabs anything from this env since it's fully + // invisible, but the evaluator does not allow for this yet. + auto inner = std::make_shared(nullptr, env.get(), 0); + for (auto from : *inheritFromExprs) + from->bindVars(es, env); + + return inner; +} + void ExprAttrs::bindVars(EvalState & es, const std::shared_ptr & env) { if (es.debugRepl) es.exprEnvs.insert(std::make_pair(this, env)); if (recursive) { - auto newEnv = std::make_shared(nullptr, env.get(), recursive ? attrs.size() : 0); + auto newEnv = [&] () -> std::shared_ptr { + auto newEnv = std::make_shared(nullptr, env.get(), attrs.size()); - Displacement displ = 0; - for (auto & i : attrs) - newEnv->vars.emplace_back(i.first, i.second.displ = displ++); + Displacement displ = 0; + for (auto & i : attrs) + newEnv->vars.emplace_back(i.first, i.second.displ = displ++); + return newEnv; + }(); // No need to sort newEnv since attrs is in sorted order. + auto inheritFromEnv = bindInheritSources(es, newEnv); for (auto & i : attrs) - i.second.e->bindVars(es, i.second.inherited ? env : newEnv); + i.second.e->bindVars(es, i.second.chooseByKind(newEnv, env, inheritFromEnv)); for (auto & i : dynamicAttrs) { i.nameExpr->bindVars(es, newEnv); @@ -351,8 +406,10 @@ void ExprAttrs::bindVars(EvalState & es, const std::shared_ptr } } else { + auto inheritFromEnv = bindInheritSources(es, env); + for (auto & i : attrs) - i.second.e->bindVars(es, env); + i.second.e->bindVars(es, i.second.chooseByKind(env, env, inheritFromEnv)); for (auto & i : dynamicAttrs) { i.nameExpr->bindVars(es, env); @@ -409,19 +466,23 @@ void ExprCall::bindVars(EvalState & es, const std::shared_ptr & void ExprLet::bindVars(EvalState & es, const std::shared_ptr & env) { - if (es.debugRepl) - es.exprEnvs.insert(std::make_pair(this, env)); + auto newEnv = [&] () -> std::shared_ptr { + auto newEnv = std::make_shared(nullptr, env.get(), attrs->attrs.size()); - auto newEnv = std::make_shared(nullptr, env.get(), attrs->attrs.size()); - - Displacement displ = 0; - for (auto & i : attrs->attrs) - newEnv->vars.emplace_back(i.first, i.second.displ = displ++); + Displacement displ = 0; + for (auto & i : attrs->attrs) + newEnv->vars.emplace_back(i.first, i.second.displ = displ++); + return newEnv; + }(); // No need to sort newEnv since attrs->attrs is in sorted order. + auto inheritFromEnv = attrs->bindInheritSources(es, newEnv); for (auto & i : attrs->attrs) - i.second.e->bindVars(es, i.second.inherited ? env : newEnv); + i.second.e->bindVars(es, i.second.chooseByKind(newEnv, env, inheritFromEnv)); + + if (es.debugRepl) + es.exprEnvs.insert(std::make_pair(this, newEnv)); body->bindVars(es, newEnv); } @@ -447,9 +508,6 @@ void ExprWith::bindVars(EvalState & es, const std::shared_ptr & break; } - if (es.debugRepl) - es.exprEnvs.insert(std::make_pair(this, env)); - attrs->bindVars(es, env); auto newEnv = std::make_shared(this, env.get()); body->bindVars(es, newEnv); diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index b6189c2a9..94356759b 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -9,110 +9,13 @@ #include "error.hh" #include "chunked-vector.hh" #include "position.hh" +#include "eval-error.hh" +#include "pos-idx.hh" +#include "pos-table.hh" namespace nix { -MakeError(EvalError, Error); -MakeError(ParseError, Error); -MakeError(AssertionError, EvalError); -MakeError(ThrownError, AssertionError); -MakeError(Abort, EvalError); -MakeError(TypeError, EvalError); -MakeError(UndefinedVarError, Error); -MakeError(MissingArgumentError, EvalError); - -class InfiniteRecursionError : public EvalError -{ - friend class EvalState; -public: - using EvalError::EvalError; -}; - -class PosIdx { - friend class PosTable; - -private: - uint32_t id; - - explicit PosIdx(uint32_t id): id(id) {} - -public: - PosIdx() : id(0) {} - - explicit operator bool() const { return id > 0; } - - bool operator <(const PosIdx other) const { return id < other.id; } - - bool operator ==(const PosIdx other) const { return id == other.id; } - - bool operator !=(const PosIdx other) const { return id != other.id; } -}; - -class PosTable -{ -public: - class Origin { - friend PosTable; - private: - // must always be invalid by default, add() replaces this with the actual value. - // subsequent add() calls use this index as a token to quickly check whether the - // current origins.back() can be reused or not. - mutable uint32_t idx = std::numeric_limits::max(); - - // Used for searching in PosTable::[]. - explicit Origin(uint32_t idx): idx(idx), origin{std::monostate()} {} - - public: - const Pos::Origin origin; - - Origin(Pos::Origin origin): origin(origin) {} - }; - - struct Offset { - uint32_t line, column; - }; - -private: - std::vector origins; - ChunkedVector offsets; - -public: - PosTable(): offsets(1024) - { - origins.reserve(1024); - } - - PosIdx add(const Origin & origin, uint32_t line, uint32_t column) - { - const auto idx = offsets.add({line, column}).second; - if (origins.empty() || origins.back().idx != origin.idx) { - origin.idx = idx; - origins.push_back(origin); - } - return PosIdx(idx + 1); - } - - Pos operator[](PosIdx p) const - { - if (p.id == 0 || p.id > offsets.size()) - return {}; - const auto idx = p.id - 1; - /* we want the last key <= idx, so we'll take prev(first key > idx). - this is guaranteed to never rewind origin.begin because the first - key is always 0. */ - const auto pastOrigin = std::upper_bound( - origins.begin(), origins.end(), Origin(idx), - [] (const auto & a, const auto & b) { return a.idx < b.idx; }); - const auto origin = *std::prev(pastOrigin); - const auto offset = offsets[idx]; - return {offset.line, offset.column, origin.origin}; - } -}; - -inline PosIdx noPos = {}; - - struct Env; struct Value; class EvalState; @@ -232,6 +135,23 @@ struct ExprVar : Expr COMMON_METHODS }; +/** + * A pseudo-expression for the purpose of evaluating the `from` expression in `inherit (from)` syntax. + * Unlike normal variable references, the displacement is set during parsing, and always refers to + * `ExprAttrs::inheritFromExprs` (by itself or in `ExprLet`), whose values are put into their own `Env`. + */ +struct ExprInheritFrom : ExprVar +{ + ExprInheritFrom(PosIdx pos, Displacement displ): ExprVar(pos, {}) + { + this->level = 0; + this->displ = displ; + this->fromWith = nullptr; + } + + void bindVars(EvalState & es, const std::shared_ptr & env); +}; + struct ExprSelect : Expr { PosIdx pos; @@ -257,16 +177,40 @@ struct ExprAttrs : Expr bool recursive; PosIdx pos; struct AttrDef { - bool inherited; + enum class Kind { + /** `attr = expr;` */ + Plain, + /** `inherit attr1 attrn;` */ + Inherited, + /** `inherit (expr) attr1 attrn;` */ + InheritedFrom, + }; + + Kind kind; Expr * e; PosIdx pos; Displacement displ; // displacement - AttrDef(Expr * e, const PosIdx & pos, bool inherited=false) - : inherited(inherited), e(e), pos(pos) { }; + AttrDef(Expr * e, const PosIdx & pos, Kind kind = Kind::Plain) + : kind(kind), e(e), pos(pos) { }; AttrDef() { }; + + template + const T & chooseByKind(const T & plain, const T & inherited, const T & inheritedFrom) const + { + switch (kind) { + case Kind::Plain: + return plain; + case Kind::Inherited: + return inherited; + default: + case Kind::InheritedFrom: + return inheritedFrom; + } + } }; typedef std::map AttrDefs; AttrDefs attrs; + std::unique_ptr> inheritFromExprs; struct DynamicAttrDef { Expr * nameExpr, * valueExpr; PosIdx pos; @@ -279,6 +223,11 @@ struct ExprAttrs : Expr ExprAttrs() : recursive(false) { }; PosIdx getPos() const override { return pos; } COMMON_METHODS + + std::shared_ptr bindInheritSources( + EvalState & es, const std::shared_ptr & env); + Env * buildInheritFromEnv(EvalState & state, Env & up); + void showBindings(const SymbolTable & symbols, std::ostream & str) const; }; struct ExprList : Expr diff --git a/src/libexpr/parser-state.hh b/src/libexpr/parser-state.hh index 0a9f076dc..34aef661f 100644 --- a/src/libexpr/parser-state.hh +++ b/src/libexpr/parser-state.hh @@ -64,17 +64,17 @@ struct ParserState inline void ParserState::dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos) { throw ParseError({ - .msg = hintfmt("attribute '%1%' already defined at %2%", + .msg = HintFmt("attribute '%1%' already defined at %2%", showAttrPath(symbols, attrPath), positions[prevPos]), - .errPos = positions[pos] + .pos = positions[pos] }); } inline void ParserState::dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos) { throw ParseError({ - .msg = hintfmt("attribute '%1%' already defined at %2%", symbols[attr], positions[prevPos]), - .errPos = positions[pos] + .msg = HintFmt("attribute '%1%' already defined at %2%", symbols[attr], positions[prevPos]), + .pos = positions[pos] }); } @@ -89,7 +89,7 @@ inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * if (i->symbol) { ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); if (j != attrs->attrs.end()) { - if (!j->second.inherited) { + if (j->second.kind != ExprAttrs::AttrDef::Kind::Inherited) { ExprAttrs * attrs2 = dynamic_cast(j->second.e); if (!attrs2) dupAttr(attrPath, pos, j->second.pos); attrs = attrs2; @@ -118,13 +118,24 @@ inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * auto ae = dynamic_cast(e); auto jAttrs = dynamic_cast(j->second.e); if (jAttrs && ae) { + if (ae->inheritFromExprs && !jAttrs->inheritFromExprs) + jAttrs->inheritFromExprs = std::make_unique>(); for (auto & ad : ae->attrs) { auto j2 = jAttrs->attrs.find(ad.first); if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error. dupAttr(ad.first, j2->second.pos, ad.second.pos); jAttrs->attrs.emplace(ad.first, ad.second); + if (ad.second.kind == ExprAttrs::AttrDef::Kind::InheritedFrom) { + auto & sel = dynamic_cast(*ad.second.e); + auto & from = dynamic_cast(*sel.e); + from.displ += jAttrs->inheritFromExprs->size(); + } } jAttrs->dynamicAttrs.insert(jAttrs->dynamicAttrs.end(), ae->dynamicAttrs.begin(), ae->dynamicAttrs.end()); + if (ae->inheritFromExprs) { + jAttrs->inheritFromExprs->insert(jAttrs->inheritFromExprs->end(), + ae->inheritFromExprs->begin(), ae->inheritFromExprs->end()); + } } else { dupAttr(attrPath, pos, j->second.pos); } @@ -154,14 +165,14 @@ inline Formals * ParserState::validateFormals(Formals * formals, PosIdx pos, Sym } if (duplicate) throw ParseError({ - .msg = hintfmt("duplicate formal function argument '%1%'", symbols[duplicate->first]), - .errPos = positions[duplicate->second] + .msg = HintFmt("duplicate formal function argument '%1%'", symbols[duplicate->first]), + .pos = positions[duplicate->second] }); if (arg && formals->has(arg)) throw ParseError({ - .msg = hintfmt("duplicate formal function argument '%1%'", symbols[arg]), - .errPos = positions[pos] + .msg = HintFmt("duplicate formal function argument '%1%'", symbols[arg]), + .pos = positions[pos] }); return formals; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index e95da37f7..b0aee7b41 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -65,8 +65,8 @@ using namespace nix; void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * error) { throw ParseError({ - .msg = hintfmt(error), - .errPos = state->positions[state->at(*loc)] + .msg = HintFmt(error), + .pos = state->positions[state->at(*loc)] }); } @@ -154,8 +154,8 @@ expr_function | LET binds IN_KW expr_function { if (!$2->dynamicAttrs.empty()) throw ParseError({ - .msg = hintfmt("dynamic attributes not allowed in let"), - .errPos = state->positions[CUR_POS] + .msg = HintFmt("dynamic attributes not allowed in let"), + .pos = state->positions[CUR_POS] }); $$ = new ExprLet($2, $4); } @@ -244,8 +244,8 @@ expr_simple static bool noURLLiterals = experimentalFeatureSettings.isEnabled(Xp::NoUrlLiterals); if (noURLLiterals) throw ParseError({ - .msg = hintfmt("URL literals are disabled"), - .errPos = state->positions[CUR_POS] + .msg = HintFmt("URL literals are disabled"), + .pos = state->positions[CUR_POS] }); $$ = new ExprString(std::string($1)); } @@ -313,17 +313,27 @@ binds if ($$->attrs.find(i.symbol) != $$->attrs.end()) state->dupAttr(i.symbol, state->at(@3), $$->attrs[i.symbol].pos); auto pos = state->at(@3); - $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, true)); + $$->attrs.emplace( + i.symbol, + ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, ExprAttrs::AttrDef::Kind::Inherited)); } delete $3; } | binds INHERIT '(' expr ')' attrs ';' { $$ = $1; - /* !!! Should ensure sharing of the expression in $4. */ + if (!$$->inheritFromExprs) + $$->inheritFromExprs = std::make_unique>(); + $$->inheritFromExprs->push_back($4); + auto from = new nix::ExprInheritFrom(state->at(@4), $$->inheritFromExprs->size() - 1); for (auto & i : *$6) { if ($$->attrs.find(i.symbol) != $$->attrs.end()) state->dupAttr(i.symbol, state->at(@6), $$->attrs[i.symbol].pos); - $$->attrs.emplace(i.symbol, ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), state->at(@6))); + $$->attrs.emplace( + i.symbol, + ExprAttrs::AttrDef( + new ExprSelect(CUR_POS, from, i.symbol), + state->at(@6), + ExprAttrs::AttrDef::Kind::InheritedFrom)); } delete $6; } @@ -340,8 +350,8 @@ attrs delete str; } else throw ParseError({ - .msg = hintfmt("dynamic attributes not allowed in inherit"), - .errPos = state->positions[state->at(@2)] + .msg = HintFmt("dynamic attributes not allowed in inherit"), + .pos = state->positions[state->at(@2)] }); } | { $$ = new AttrPath; } diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc index 099607638..50d0d9895 100644 --- a/src/libexpr/paths.cc +++ b/src/libexpr/paths.cc @@ -1,5 +1,4 @@ #include "eval.hh" -#include "fs-input-accessor.hh" namespace nix { @@ -8,4 +7,9 @@ SourcePath EvalState::rootPath(CanonPath path) return {rootFS, std::move(path)}; } +SourcePath EvalState::rootPath(PathView path) +{ + return {rootFS, CanonPath(absPath(path))}; +} + } diff --git a/src/libexpr/pos-idx.hh b/src/libexpr/pos-idx.hh new file mode 100644 index 000000000..9949f1dc5 --- /dev/null +++ b/src/libexpr/pos-idx.hh @@ -0,0 +1,48 @@ +#pragma once + +#include + +namespace nix { + +class PosIdx +{ + friend class PosTable; + +private: + uint32_t id; + + explicit PosIdx(uint32_t id) + : id(id) + { + } + +public: + PosIdx() + : id(0) + { + } + + explicit operator bool() const + { + return id > 0; + } + + bool operator<(const PosIdx other) const + { + return id < other.id; + } + + bool operator==(const PosIdx other) const + { + return id == other.id; + } + + bool operator!=(const PosIdx other) const + { + return id != other.id; + } +}; + +inline PosIdx noPos = {}; + +} diff --git a/src/libexpr/pos-table.hh b/src/libexpr/pos-table.hh new file mode 100644 index 000000000..1decf3c85 --- /dev/null +++ b/src/libexpr/pos-table.hh @@ -0,0 +1,83 @@ +#pragma once + +#include +#include +#include + +#include "chunked-vector.hh" +#include "pos-idx.hh" +#include "position.hh" + +namespace nix { + +class PosTable +{ +public: + class Origin + { + friend PosTable; + private: + // must always be invalid by default, add() replaces this with the actual value. + // subsequent add() calls use this index as a token to quickly check whether the + // current origins.back() can be reused or not. + mutable uint32_t idx = std::numeric_limits::max(); + + // Used for searching in PosTable::[]. + explicit Origin(uint32_t idx) + : idx(idx) + , origin{std::monostate()} + { + } + + public: + const Pos::Origin origin; + + Origin(Pos::Origin origin) + : origin(origin) + { + } + }; + + struct Offset + { + uint32_t line, column; + }; + +private: + std::vector origins; + ChunkedVector offsets; + +public: + PosTable() + : offsets(1024) + { + origins.reserve(1024); + } + + PosIdx add(const Origin & origin, uint32_t line, uint32_t column) + { + const auto idx = offsets.add({line, column}).second; + if (origins.empty() || origins.back().idx != origin.idx) { + origin.idx = idx; + origins.push_back(origin); + } + return PosIdx(idx + 1); + } + + Pos operator[](PosIdx p) const + { + if (p.id == 0 || p.id > offsets.size()) + return {}; + const auto idx = p.id - 1; + /* we want the last key <= idx, so we'll take prev(first key > idx). + this is guaranteed to never rewind origin.begin because the first + key is always 0. */ + const auto pastOrigin = std::upper_bound( + origins.begin(), origins.end(), Origin(idx), [](const auto & a, const auto & b) { return a.idx < b.idx; }); + const auto origin = *std::prev(pastOrigin); + const auto offset = offsets[idx]; + return {offset.line, offset.column, origin.origin}; + } +}; + +} diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 1197b6e13..78f7f71ed 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -39,10 +39,6 @@ namespace nix { * Miscellaneous *************************************************************/ - -InvalidPathError::InvalidPathError(const Path & path) : - EvalError("path '%s' is not valid", path), path(path) {} - StringMap EvalState::realiseContext(const NixStringContext & context) { std::vector drvs; @@ -51,7 +47,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context) for (auto & c : context) { auto ensureValid = [&](const StorePath & p) { if (!store->isValidPath(p)) - debugThrowLastTrace(InvalidPathError(store->printStorePath(p))); + error(store->printStorePath(p)).debugThrow(); }; std::visit(overloaded { [&](const NixStringContextElem::Built & b) { @@ -78,9 +74,10 @@ StringMap EvalState::realiseContext(const NixStringContext & context) if (drvs.empty()) return {}; if (!evalSettings.enableImportFromDerivation) - debugThrowLastTrace(Error( + error( "cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled", - drvs.begin()->to_string(*store))); + drvs.begin()->to_string(*store) + ).debugThrow(); /* Build/substitute the context. */ std::vector buildReqs; @@ -118,7 +115,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context) return res; } -static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, bool resolveSymlinks = true) +static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, std::optional resolveSymlinks = SymlinkResolution::Full) { NixStringContext context; @@ -130,7 +127,7 @@ static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, bo auto realPath = state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context); path = {path.accessor, CanonPath(realPath)}; } - return resolveSymlinks ? path.resolveSymlinks() : path; + return resolveSymlinks ? path.resolveSymlinks(*resolveSymlinks) : path; } catch (Error & e) { e.addTrace(state.positions[pos], "while realising the context of path '%s'", path); throw; @@ -170,7 +167,7 @@ static void mkOutputString( argument. */ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * vScope, Value & v) { - auto path = realisePath(state, pos, vPath, false); + auto path = realisePath(state, pos, vPath, std::nullopt); auto path2 = path.path.abs(); // FIXME @@ -340,16 +337,16 @@ void prim_importNative(EvalState & state, const PosIdx pos, Value * * args, Valu void *handle = dlopen(path.path.c_str(), RTLD_LAZY | RTLD_LOCAL); if (!handle) - state.debugThrowLastTrace(EvalError("could not open '%1%': %2%", path, dlerror())); + state.error("could not open '%1%': %2%", path, dlerror()).debugThrow(); dlerror(); ValueInitializer func = (ValueInitializer) dlsym(handle, sym.c_str()); if(!func) { char *message = dlerror(); if (message) - state.debugThrowLastTrace(EvalError("could not load symbol '%1%' from '%2%': %3%", sym, path, message)); + state.error("could not load symbol '%1%' from '%2%': %3%", sym, path, message).debugThrow(); else - state.debugThrowLastTrace(EvalError("symbol '%1%' from '%2%' resolved to NULL when a function pointer was expected", sym, path)); + state.error("symbol '%1%' from '%2%' resolved to NULL when a function pointer was expected", sym, path).debugThrow(); } (func)(state, v); @@ -365,7 +362,7 @@ void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v) auto elems = args[0]->listElems(); auto count = args[0]->listSize(); if (count == 0) - state.error("at least one argument to 'exec' required").atPos(pos).debugThrow(); + state.error("at least one argument to 'exec' required").atPos(pos).debugThrow(); NixStringContext context; auto program = state.coerceToString(pos, *elems[0], context, "while evaluating the first element of the argument passed to builtins.exec", @@ -380,7 +377,7 @@ void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v) try { auto _ = state.realiseContext(context); // FIXME: Handle CA derivations } catch (InvalidPathError & e) { - state.error("cannot execute '%1%', since path '%2%' is not valid", program, e.path).atPos(pos).debugThrow(); + state.error("cannot execute '%1%', since path '%2%' is not valid", program, e.path).atPos(pos).debugThrow(); } auto output = runProgram(program, true, commandArgs); @@ -582,7 +579,7 @@ struct CompareValues if (v1->type() == nInt && v2->type() == nFloat) return v1->integer < v2->fpoint; if (v1->type() != v2->type()) - state.error("cannot compare %s with %s", showType(*v1), showType(*v2)).debugThrow(); + state.error("cannot compare %s with %s", showType(*v1), showType(*v2)).debugThrow(); // Allow selecting a subset of enum values #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wswitch-enum" @@ -610,7 +607,7 @@ struct CompareValues } } default: - state.error("cannot compare %s with %s; values of that type are incomparable", showType(*v1), showType(*v2)).debugThrow(); + state.error("cannot compare %s with %s; values of that type are incomparable", showType(*v1), showType(*v2)).debugThrow(); #pragma GCC diagnostic pop } } catch (Error & e) { @@ -637,7 +634,7 @@ static Bindings::iterator getAttr( { Bindings::iterator value = attrSet->find(attrSym); if (value == attrSet->end()) { - state.error("attribute '%s' missing", state.symbols[attrSym]).withTrace(noPos, errorCtx).debugThrow(); + state.error("attribute '%s' missing", state.symbols[attrSym]).withTrace(noPos, errorCtx).debugThrow(); } return value; } @@ -708,38 +705,53 @@ static RegisterPrimOp primop_genericClosure(PrimOp { .args = {"attrset"}, .arity = 1, .doc = R"( - Take an *attrset* with values named `startSet` and `operator` in order to - return a *list of attrsets* by starting with the `startSet` and recursively - applying the `operator` function to each `item`. The *attrsets* in the - `startSet` and the *attrsets* produced by `operator` must contain a value - named `key` which is comparable. The result is produced by calling `operator` - for each `item` with a value for `key` that has not been called yet including - newly produced `item`s. The function terminates when no new `item`s are - produced. The resulting *list of attrsets* contains only *attrsets* with a - unique key. For example, + `builtins.genericClosure` iteratively computes the transitive closure over an arbitrary relation defined by a function. - ``` - builtins.genericClosure { - startSet = [ {key = 5;} ]; - operator = item: [{ - key = if (item.key / 2 ) * 2 == item.key - then item.key / 2 - else 3 * item.key + 1; - }]; - } - ``` - evaluates to - ``` - [ { key = 5; } { key = 16; } { key = 8; } { key = 4; } { key = 2; } { key = 1; } ] - ``` + It takes *attrset* with two attributes named `startSet` and `operator`, and returns a list of attrbute sets: + + - `startSet`: + The initial list of attribute sets. + + - `operator`: + A function that takes an attribute set and returns a list of attribute sets. + It defines how each item in the current set is processed and expanded into more items. + + Each attribute set in the list `startSet` and the list returned by `operator` must have an attribute `key`, which must support equality comparison. + The value of `key` can be one of the following types: - `key` can be one of the following types: - [Number](@docroot@/language/values.md#type-number) - [Boolean](@docroot@/language/values.md#type-boolean) - [String](@docroot@/language/values.md#type-string) - [Path](@docroot@/language/values.md#type-path) - [List](@docroot@/language/values.md#list) + The result is produced by calling the `operator` on each `item` that has not been called yet, including newly added items, until no new items are added. + Items are compared by their `key` attribute. + + Common usages are: + + - Generating unique collections of items, such as dependency graphs. + - Traversing through structures that may contain cycles or loops. + - Processing data structures with complex internal relationships. + + > **Example** + > + > ```nix + > builtins.genericClosure { + > startSet = [ {key = 5;} ]; + > operator = item: [{ + > key = if (item.key / 2 ) * 2 == item.key + > then item.key / 2 + > else 3 * item.key + 1; + > }]; + > } + > ``` + > + > evaluates to + > + > ```nix + > [ { key = 5; } { key = 16; } { key = 8; } { key = 4; } { key = 2; } { key = 1; } ] + > ``` )", .fun = prim_genericClosure, }); @@ -757,21 +769,12 @@ static RegisterPrimOp primop_break({ if (state.debugRepl && !state.debugTraces.empty()) { auto error = Error(ErrorInfo { .level = lvlInfo, - .msg = hintfmt("breakpoint reached"), - .errPos = state.positions[pos], + .msg = HintFmt("breakpoint reached"), + .pos = state.positions[pos], }); auto & dt = state.debugTraces.front(); state.runDebugRepl(&error, dt.env, dt.expr); - - if (state.debugQuit) { - // If the user elects to quit the repl, throw an exception. - throw Error(ErrorInfo{ - .level = lvlInfo, - .msg = hintfmt("quit the debugger"), - .errPos = nullptr, - }); - } } // Return the value we were passed. @@ -790,7 +793,7 @@ static RegisterPrimOp primop_abort({ NixStringContext context; auto s = state.coerceToString(pos, *args[0], context, "while evaluating the error message passed to builtins.abort").toOwned(); - state.debugThrowLastTrace(Abort("evaluation aborted with the following error message: '%1%'", s)); + state.error("evaluation aborted with the following error message: '%1%'", s).debugThrow(); } }); @@ -809,7 +812,7 @@ static RegisterPrimOp primop_throw({ NixStringContext context; auto s = state.coerceToString(pos, *args[0], context, "while evaluating the error message passed to builtin.throw").toOwned(); - state.debugThrowLastTrace(ThrownError(s)); + state.error(s).debugThrow(); } }); @@ -823,7 +826,7 @@ static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value * * auto message = state.coerceToString(pos, *args[0], context, "while evaluating the error message passed to builtins.addErrorContext", false, false).toOwned(); - e.addTrace(nullptr, hintfmt(message), true); + e.addTrace(nullptr, HintFmt(message)); throw; } } @@ -882,7 +885,7 @@ static void prim_tryEval(EvalState & state, const PosIdx pos, Value * * args, Va /* increment state.trylevel, and decrement it when this function returns. */ MaintainCount trylevel(state.trylevel); - void (* savedDebugRepl)(ref es, const ValMap & extraEnv) = nullptr; + ReplExitStatus (* savedDebugRepl)(ref es, const ValMap & extraEnv) = nullptr; if (state.debugRepl && evalSettings.ignoreExceptionsDuringTry) { /* to prevent starting the repl from exceptions withing a tryEval, null it. */ @@ -998,6 +1001,10 @@ static void prim_trace(EvalState & state, const PosIdx pos, Value * * args, Valu printError("trace: %1%", args[0]->string_view()); else printError("trace: %1%", ValuePrinter(state, *args[0])); + if (evalSettings.builtinsTraceDebugger && state.debugRepl && !state.debugTraces.empty()) { + const DebugTrace & last = state.debugTraces.front(); + state.runDebugRepl(nullptr, last.env, last.expr); + } state.forceValue(*args[1], pos); v = *args[1]; } @@ -1009,6 +1016,12 @@ static RegisterPrimOp primop_trace({ Evaluate *e1* and print its abstract syntax representation on standard error. Then return *e2*. This function is useful for debugging. + + If the + [`debugger-on-trace`](@docroot@/command-ref/conf-file.md#conf-debugger-on-trace) + option is set to `true` and the `--debugger` flag is given, the + interactive debugger will be started when `trace` is called (like + [`break`](@docroot@/language/builtins.md#builtins-break)). )", .fun = prim_trace, }); @@ -1074,10 +1087,10 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * * often results from the composition of several functions * (derivationStrict, derivation, mkDerivation, mkPythonModule, etc.) */ - e.addTrace(nullptr, hintfmt( + e.addTrace(nullptr, HintFmt( "while evaluating derivation '%s'\n" " whose name attribute is located at %s", - drvName, pos), true); + drvName, pos)); throw; } } @@ -1088,9 +1101,10 @@ drvName, Bindings * attrs, Value & v) /* Check whether attributes should be passed as a JSON file. */ using nlohmann::json; std::optional jsonObject; + auto pos = v.determinePos(noPos); auto attr = attrs->find(state.sStructuredAttrs); if (attr != attrs->end() && - state.forceBool(*attr->value, noPos, + state.forceBool(*attr->value, pos, "while evaluating the `__structuredAttrs` " "attribute passed to builtins.derivationStrict")) jsonObject = json::object(); @@ -1099,7 +1113,7 @@ drvName, Bindings * attrs, Value & v) bool ignoreNulls = false; attr = attrs->find(state.sIgnoreNulls); if (attr != attrs->end()) - ignoreNulls = state.forceBool(*attr->value, noPos, "while evaluating the `__ignoreNulls` attribute " "passed to builtins.derivationStrict"); + ignoreNulls = state.forceBool(*attr->value, pos, "while evaluating the `__ignoreNulls` attribute " "passed to builtins.derivationStrict"); /* Build the derivation expression by processing the attributes. */ Derivation drv; @@ -1124,41 +1138,40 @@ drvName, Bindings * attrs, Value & v) auto handleHashMode = [&](const std::string_view s) { if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive; else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat; - else if (s == "text") { + else if (s == "git") { + experimentalFeatureSettings.require(Xp::GitHashing); + ingestionMethod = FileIngestionMethod::Git; + } else if (s == "text") { experimentalFeatureSettings.require(Xp::DynamicDerivations); ingestionMethod = TextIngestionMethod {}; } else - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s), - .errPos = state.positions[noPos] - })); + state.error( + "invalid value '%s' for 'outputHashMode' attribute", s + ).atPos(v).debugThrow(); }; auto handleOutputs = [&](const Strings & ss) { outputs.clear(); for (auto & j : ss) { if (outputs.find(j) != outputs.end()) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("duplicate derivation output '%1%'", j), - .errPos = state.positions[noPos] - })); + state.error("duplicate derivation output '%1%'", j) + .atPos(v) + .debugThrow(); /* !!! Check whether j is a valid attribute name. */ /* Derivations cannot be named ‘drv’, because then we'd have an attribute ‘drvPath’ in the resulting set. */ if (j == "drv") - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("invalid derivation output name 'drv'" ), - .errPos = state.positions[noPos] - })); + state.error("invalid derivation output name 'drv'") + .atPos(v) + .debugThrow(); outputs.insert(j); } if (outputs.empty()) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("derivation cannot have an empty set of outputs"), - .errPos = state.positions[noPos] - })); + state.error("derivation cannot have an empty set of outputs") + .atPos(v) + .debugThrow(); }; try { @@ -1167,16 +1180,16 @@ drvName, Bindings * attrs, Value & v) const std::string_view context_below(""); if (ignoreNulls) { - state.forceValue(*i->value, noPos); + state.forceValue(*i->value, pos); if (i->value->type() == nNull) continue; } - if (i->name == state.sContentAddressed && state.forceBool(*i->value, noPos, context_below)) { + if (i->name == state.sContentAddressed && state.forceBool(*i->value, pos, context_below)) { contentAddressed = true; experimentalFeatureSettings.require(Xp::CaDerivations); } - else if (i->name == state.sImpure && state.forceBool(*i->value, noPos, context_below)) { + else if (i->name == state.sImpure && state.forceBool(*i->value, pos, context_below)) { isImpure = true; experimentalFeatureSettings.require(Xp::ImpureDerivations); } @@ -1184,9 +1197,9 @@ drvName, Bindings * attrs, Value & v) /* The `args' attribute is special: it supplies the command-line arguments to the builder. */ else if (i->name == state.sArgs) { - state.forceList(*i->value, noPos, context_below); + state.forceList(*i->value, pos, context_below); for (auto elem : i->value->listItems()) { - auto s = state.coerceToString(noPos, *elem, context, + auto s = state.coerceToString(pos, *elem, context, "while evaluating an element of the argument list", true).toOwned(); drv.args.push_back(s); @@ -1201,29 +1214,29 @@ drvName, Bindings * attrs, Value & v) if (i->name == state.sStructuredAttrs) continue; - (*jsonObject)[key] = printValueAsJSON(state, true, *i->value, noPos, context); + (*jsonObject)[key] = printValueAsJSON(state, true, *i->value, pos, context); if (i->name == state.sBuilder) - drv.builder = state.forceString(*i->value, context, noPos, context_below); + drv.builder = state.forceString(*i->value, context, pos, context_below); else if (i->name == state.sSystem) - drv.platform = state.forceStringNoCtx(*i->value, noPos, context_below); + drv.platform = state.forceStringNoCtx(*i->value, pos, context_below); else if (i->name == state.sOutputHash) - outputHash = state.forceStringNoCtx(*i->value, noPos, context_below); + outputHash = state.forceStringNoCtx(*i->value, pos, context_below); else if (i->name == state.sOutputHashAlgo) - outputHashAlgo = state.forceStringNoCtx(*i->value, noPos, context_below); + outputHashAlgo = state.forceStringNoCtx(*i->value, pos, context_below); else if (i->name == state.sOutputHashMode) - handleHashMode(state.forceStringNoCtx(*i->value, noPos, context_below)); + handleHashMode(state.forceStringNoCtx(*i->value, pos, context_below)); else if (i->name == state.sOutputs) { /* Require ‘outputs’ to be a list of strings. */ - state.forceList(*i->value, noPos, context_below); + state.forceList(*i->value, pos, context_below); Strings ss; for (auto elem : i->value->listItems()) - ss.emplace_back(state.forceStringNoCtx(*elem, noPos, context_below)); + ss.emplace_back(state.forceStringNoCtx(*elem, pos, context_below)); handleOutputs(ss); } } else { - auto s = state.coerceToString(noPos, *i->value, context, context_below, true).toOwned(); + auto s = state.coerceToString(pos, *i->value, context, context_below, true).toOwned(); drv.env.emplace(key, s); if (i->name == state.sBuilder) drv.builder = std::move(s); else if (i->name == state.sSystem) drv.platform = std::move(s); @@ -1238,8 +1251,7 @@ drvName, Bindings * attrs, Value & v) } catch (Error & e) { e.addTrace(state.positions[i->pos], - hintfmt("while evaluating attribute '%1%' of derivation '%2%'", key, drvName), - true); + HintFmt("while evaluating attribute '%1%' of derivation '%2%'", key, drvName)); throw; } } @@ -1281,16 +1293,14 @@ drvName, Bindings * attrs, Value & v) /* Do we have all required attributes? */ if (drv.builder == "") - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("required attribute 'builder' missing"), - .errPos = state.positions[noPos] - })); + state.error("required attribute 'builder' missing") + .atPos(v) + .debugThrow(); if (drv.platform == "") - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("required attribute 'system' missing"), - .errPos = state.positions[noPos] - })); + state.error("required attribute 'system' missing") + .atPos(v) + .debugThrow(); /* Check whether the derivation name is valid. */ if (isDerivation(drvName) && @@ -1298,10 +1308,10 @@ drvName, Bindings * attrs, Value & v) outputs.size() == 1 && *(outputs.begin()) == "out")) { - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("derivation names are allowed to end in '%s' only if they produce a single derivation file", drvExtension), - .errPos = state.positions[noPos] - })); + state.error( + "derivation names are allowed to end in '%s' only if they produce a single derivation file", + drvExtension + ).atPos(v).debugThrow(); } if (outputHash) { @@ -1310,10 +1320,9 @@ drvName, Bindings * attrs, Value & v) Ignore `__contentAddressed` because fixed output derivations are already content addressed. */ if (outputs.size() != 1 || *(outputs.begin()) != "out") - state.debugThrowLastTrace(Error({ - .msg = hintfmt("multiple outputs are not supported in fixed-output derivations"), - .errPos = state.positions[noPos] - })); + state.error( + "multiple outputs are not supported in fixed-output derivations" + ).atPos(v).debugThrow(); auto h = newHashAllowEmpty(*outputHash, parseHashAlgoOpt(outputHashAlgo)); @@ -1332,10 +1341,8 @@ drvName, Bindings * attrs, Value & v) else if (contentAddressed || isImpure) { if (contentAddressed && isImpure) - throw EvalError({ - .msg = hintfmt("derivation cannot be both content-addressed and impure"), - .errPos = state.positions[noPos] - }); + state.error("derivation cannot be both content-addressed and impure") + .atPos(v).debugThrow(); auto ha = parseHashAlgoOpt(outputHashAlgo).value_or(HashAlgorithm::SHA256); auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive); @@ -1376,10 +1383,10 @@ drvName, Bindings * attrs, Value & v) for (auto & i : outputs) { auto h = get(hashModulo.hashes, i); if (!h) - throw AssertionError({ - .msg = hintfmt("derivation produced no hash for output '%s'", i), - .errPos = state.positions[noPos], - }); + state.error( + "derivation produced no hash for output '%s'", + i + ).atPos(v).debugThrow(); auto outPath = state.store->makeOutputPath(i, *h, drvName); drv.env[i] = state.store->printStorePath(outPath); drv.outputs.insert_or_assign( @@ -1485,10 +1492,10 @@ static RegisterPrimOp primop_toPath({ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args, Value & v) { if (evalSettings.pureEval) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("'%s' is not allowed in pure evaluation mode", "builtins.storePath"), - .errPos = state.positions[pos] - })); + state.error( + "'%s' is not allowed in pure evaluation mode", + "builtins.storePath" + ).atPos(pos).debugThrow(); NixStringContext context; auto path = state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'").path; @@ -1498,10 +1505,8 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args, if (!state.store->isStorePath(path.abs())) path = CanonPath(canonPath(path.abs(), true)); if (!state.store->isInStore(path.abs())) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("path '%1%' is not in the Nix store", path), - .errPos = state.positions[pos] - })); + state.error("path '%1%' is not in the Nix store", path) + .atPos(pos).debugThrow(); auto path2 = state.store->toStorePath(path.abs()).first; if (!settings.readOnlyMode) state.store->ensurePath(path2); @@ -1534,13 +1539,16 @@ static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args, try { auto & arg = *args[0]; - auto path = realisePath(state, pos, arg); - /* SourcePath doesn't know about trailing slash. */ + state.forceValue(arg, pos); auto mustBeDir = arg.type() == nString && (arg.string_view().ends_with("/") || arg.string_view().ends_with("/.")); + auto symlinkResolution = + mustBeDir ? SymlinkResolution::Full : SymlinkResolution::Ancestors; + auto path = realisePath(state, pos, arg, symlinkResolution); + auto st = path.maybeLstat(); auto exists = st && (!mustBeDir || st->type == SourceAccessor::tDirectory); v.mkBool(exists); @@ -1616,7 +1624,10 @@ static void prim_readFile(EvalState & state, const PosIdx pos, Value * * args, V auto path = realisePath(state, pos, *args[0]); auto s = path.readFile(); if (s.find((char) 0) != std::string::npos) - state.debugThrowLastTrace(Error("the contents of the file '%1%' cannot be represented as a Nix string", path)); + state.error( + "the contents of the file '%1%' cannot be represented as a Nix string", + path + ).atPos(pos).debugThrow(); StorePathSet refs; if (state.store->isInStore(path.path.abs())) { try { @@ -1673,10 +1684,11 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V auto rewrites = state.realiseContext(context); path = rewriteStrings(path, rewrites); } catch (InvalidPathError & e) { - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("cannot find '%1%', since path '%2%' is not valid", path, e.path), - .errPos = state.positions[pos] - })); + state.error( + "cannot find '%1%', since path '%2%' is not valid", + path, + e.path + ).atPos(pos).debugThrow(); } searchPath.elements.emplace_back(SearchPath::Elem { @@ -1745,10 +1757,7 @@ static void prim_hashFile(EvalState & state, const PosIdx pos, Value * * args, V auto algo = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashFile"); std::optional ha = parseHashAlgo(algo); if (!ha) - state.debugThrowLastTrace(Error({ - .msg = hintfmt("unknown hash algo '%1%'", algo), - .errPos = state.positions[pos] - })); + state.error("unknown hash algorithm '%1%'", algo).atPos(pos).debugThrow(); auto path = realisePath(state, pos, *args[1]); @@ -1777,7 +1786,7 @@ static std::string_view fileTypeToString(InputAccessor::Type type) static void prim_readFileType(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto path = realisePath(state, pos, *args[0], false); + auto path = realisePath(state, pos, *args[0], std::nullopt); /* Retrieve the directory entry type and stringize it. */ v.mkString(fileTypeToString(path.lstat().type)); } @@ -1816,7 +1825,7 @@ static void prim_readDir(EvalState & state, const PosIdx pos, Value * * args, Va // detailed node info quickly in this case we produce a thunk to // query the file type lazily. auto epath = state.allocValue(); - epath->mkPath(path + name); + epath->mkPath(path / name); if (!readFileType) readFileType = &state.getBuiltin("readFileType"); attr.mkApp(readFileType, epath); @@ -2068,13 +2077,12 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Val if (auto p = std::get_if(&c.raw)) refs.insert(p->path); else - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt( - "in 'toFile': the file named '%1%' must not contain a reference " - "to a derivation but contains (%2%)", - name, c.to_string()), - .errPos = state.positions[pos] - })); + state.error( + "files created by %1% may not reference derivations, but %2% references %3%", + "builtins.toFile", + name, + c.to_string() + ).atPos(pos).debugThrow(); } auto storePath = settings.readOnlyMode @@ -2084,7 +2092,7 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Val }) : ({ StringSource s { contents }; - state.store->addToStoreFromDump(s, name, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, state.repair); + state.store->addToStoreFromDump(s, name, FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, state.repair); }); /* Note: we don't need to add `context' to the context of the @@ -2241,9 +2249,19 @@ static void addPath( }); if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { - auto dstPath = fetchToStore(*state.store, path.resolveSymlinks(), name, method, filter.get(), state.repair); + auto dstPath = fetchToStore( + *state.store, + path.resolveSymlinks(), + settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, + name, + method, + filter.get(), + state.repair); if (expectedHash && expectedStorePath != dstPath) - state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path)); + state.error( + "store path mismatch in (possibly filtered) path added from '%s'", + path + ).atPos(pos).debugThrow(); state.allowAndSetStorePathString(dstPath, v); } else state.allowAndSetStorePathString(*expectedStorePath, v); @@ -2343,16 +2361,15 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value else if (n == "sha256") expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `sha256` attribute passed to builtins.path"), HashAlgorithm::SHA256); else - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("unsupported argument '%1%' to 'addPath'", state.symbols[attr.name]), - .errPos = state.positions[attr.pos] - })); + state.error( + "unsupported argument '%1%' to 'addPath'", + state.symbols[attr.name] + ).atPos(attr.pos).debugThrow(); } if (!path) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("missing required 'path' attribute in the first argument to builtins.path"), - .errPos = state.positions[pos] - })); + state.error( + "missing required 'path' attribute in the first argument to builtins.path" + ).atPos(pos).debugThrow(); if (name.empty()) name = path->baseName(); @@ -2770,10 +2787,7 @@ static void prim_functionArgs(EvalState & state, const PosIdx pos, Value * * arg return; } if (!args[0]->isLambda()) - state.debugThrowLastTrace(TypeError({ - .msg = hintfmt("'functionArgs' requires a function"), - .errPos = state.positions[pos] - })); + state.error("'functionArgs' requires a function").atPos(pos).debugThrow(); if (!args[0]->lambda.fun->hasFormals()) { v.mkAttrs(&state.emptyBindings); @@ -2943,10 +2957,10 @@ static void elemAt(EvalState & state, const PosIdx pos, Value & list, int n, Val { state.forceList(list, pos, "while evaluating the first argument passed to builtins.elemAt"); if (n < 0 || (unsigned int) n >= list.listSize()) - state.debugThrowLastTrace(Error({ - .msg = hintfmt("list index %1% is out of bounds", n), - .errPos = state.positions[pos] - })); + state.error( + "list index %1% is out of bounds", + n + ).atPos(pos).debugThrow(); state.forceValue(*list.listElems()[n], pos); v = *list.listElems()[n]; } @@ -2991,10 +3005,7 @@ static void prim_tail(EvalState & state, const PosIdx pos, Value * * args, Value { state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.tail"); if (args[0]->listSize() == 0) - state.debugThrowLastTrace(Error({ - .msg = hintfmt("'tail' called on an empty list"), - .errPos = state.positions[pos] - })); + state.error("'tail' called on an empty list").atPos(pos).debugThrow(); state.mkList(v, args[0]->listSize() - 1); for (unsigned int n = 0; n < v.listSize(); ++n) @@ -3251,7 +3262,7 @@ static void prim_genList(EvalState & state, const PosIdx pos, Value * * args, Va auto len = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.genList"); if (len < 0) - state.error("cannot create list of size %1%", len).debugThrow(); + state.error("cannot create list of size %1%", len).atPos(pos).debugThrow(); // More strict than striclty (!) necessary, but acceptable // as evaluating map without accessing any values makes little sense. @@ -3568,10 +3579,7 @@ static void prim_div(EvalState & state, const PosIdx pos, Value * * args, Value NixFloat f2 = state.forceFloat(*args[1], pos, "while evaluating the second operand of the division"); if (f2 == 0) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("division by zero"), - .errPos = state.positions[pos] - })); + state.error("division by zero").atPos(pos).debugThrow(); if (args[0]->type() == nFloat || args[1]->type() == nFloat) { v.mkFloat(state.forceFloat(*args[0], pos, "while evaluating the first operand of the division") / f2); @@ -3580,10 +3588,7 @@ static void prim_div(EvalState & state, const PosIdx pos, Value * * args, Value NixInt i2 = state.forceInt(*args[1], pos, "while evaluating the second operand of the division"); /* Avoid division overflow as it might raise SIGFPE. */ if (i1 == std::numeric_limits::min() && i2 == -1) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("overflow in integer division"), - .errPos = state.positions[pos] - })); + state.error("overflow in integer division").atPos(pos).debugThrow(); v.mkInt(i1 / i2); } @@ -3714,10 +3719,7 @@ static void prim_substring(EvalState & state, const PosIdx pos, Value * * args, int start = state.forceInt(*args[0], pos, "while evaluating the first argument (the start offset) passed to builtins.substring"); if (start < 0) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("negative start position in 'substring'"), - .errPos = state.positions[pos] - })); + state.error("negative start position in 'substring'").atPos(pos).debugThrow(); int len = state.forceInt(*args[1], pos, "while evaluating the second argument (the substring length) passed to builtins.substring"); @@ -3782,10 +3784,7 @@ static void prim_hashString(EvalState & state, const PosIdx pos, Value * * args, auto algo = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashString"); std::optional ha = parseHashAlgo(algo); if (!ha) - state.debugThrowLastTrace(Error({ - .msg = hintfmt("unknown hash algo '%1%'", algo), - .errPos = state.positions[pos] - })); + state.error("unknown hash algorithm '%1%'", algo).atPos(pos).debugThrow(); NixStringContext context; // discarded auto s = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString"); @@ -3951,15 +3950,13 @@ void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v) } catch (std::regex_error & e) { if (e.code() == std::regex_constants::error_space) { // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("memory limit exceeded by regular expression '%s'", re), - .errPos = state.positions[pos] - })); + state.error("memory limit exceeded by regular expression '%s'", re) + .atPos(pos) + .debugThrow(); } else - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("invalid regular expression '%s'", re), - .errPos = state.positions[pos] - })); + state.error("invalid regular expression '%s'", re) + .atPos(pos) + .debugThrow(); } } @@ -4055,15 +4052,13 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v) } catch (std::regex_error & e) { if (e.code() == std::regex_constants::error_space) { // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("memory limit exceeded by regular expression '%s'", re), - .errPos = state.positions[pos] - })); + state.error("memory limit exceeded by regular expression '%s'", re) + .atPos(pos) + .debugThrow(); } else - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("invalid regular expression '%s'", re), - .errPos = state.positions[pos] - })); + state.error("invalid regular expression '%s'", re) + .atPos(pos) + .debugThrow(); } } @@ -4139,7 +4134,9 @@ static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value * * a state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.replaceStrings"); state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.replaceStrings"); if (args[0]->listSize() != args[1]->listSize()) - state.error("'from' and 'to' arguments passed to builtins.replaceStrings have different lengths").atPos(pos).debugThrow(); + state.error( + "'from' and 'to' arguments passed to builtins.replaceStrings have different lengths" + ).atPos(pos).debugThrow(); std::vector from; from.reserve(args[0]->listSize()); diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index db940f277..1eec8b316 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -98,30 +98,30 @@ static void prim_addDrvOutputDependencies(EvalState & state, const PosIdx pos, V auto contextSize = context.size(); if (contextSize != 1) { - throw EvalError({ - .msg = hintfmt("context of string '%s' must have exactly one element, but has %d", *s, contextSize), - .errPos = state.positions[pos] - }); + state.error( + "context of string '%s' must have exactly one element, but has %d", + *s, + contextSize + ).atPos(pos).debugThrow(); } NixStringContext context2 { (NixStringContextElem { std::visit(overloaded { [&](const NixStringContextElem::Opaque & c) -> NixStringContextElem::DrvDeep { if (!c.path.isDerivation()) { - throw EvalError({ - .msg = hintfmt("path '%s' is not a derivation", - state.store->printStorePath(c.path)), - .errPos = state.positions[pos], - }); + state.error( + "path '%s' is not a derivation", + state.store->printStorePath(c.path) + ).atPos(pos).debugThrow(); } return NixStringContextElem::DrvDeep { .drvPath = c.path, }; }, [&](const NixStringContextElem::Built & c) -> NixStringContextElem::DrvDeep { - throw EvalError({ - .msg = hintfmt("`addDrvOutputDependencies` can only act on derivations, not on a derivation output such as '%1%'", c.output), - .errPos = state.positions[pos], - }); + state.error( + "`addDrvOutputDependencies` can only act on derivations, not on a derivation output such as '%1%'", + c.output + ).atPos(pos).debugThrow(); }, [&](const NixStringContextElem::DrvDeep & c) -> NixStringContextElem::DrvDeep { /* Reuse original item because we want this to be idempotent. */ @@ -261,10 +261,10 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar for (auto & i : *args[1]->attrs) { const auto & name = state.symbols[i.name]; if (!state.store->isStorePath(name)) - throw EvalError({ - .msg = hintfmt("context key '%s' is not a store path", name), - .errPos = state.positions[i.pos] - }); + state.error( + "context key '%s' is not a store path", + name + ).atPos(i.pos).debugThrow(); auto namePath = state.store->parseStorePath(name); if (!settings.readOnlyMode) state.store->ensurePath(namePath); @@ -281,10 +281,10 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar if (iter != i.value->attrs->end()) { if (state.forceBool(*iter->value, iter->pos, "while evaluating the `allOutputs` attribute of a string context")) { if (!isDerivation(name)) { - throw EvalError({ - .msg = hintfmt("tried to add all-outputs context of %s, which is not a derivation, to a string", name), - .errPos = state.positions[i.pos] - }); + state.error( + "tried to add all-outputs context of %s, which is not a derivation, to a string", + name + ).atPos(i.pos).debugThrow(); } context.emplace(NixStringContextElem::DrvDeep { .drvPath = namePath, @@ -296,10 +296,10 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar if (iter != i.value->attrs->end()) { state.forceList(*iter->value, iter->pos, "while evaluating the `outputs` attribute of a string context"); if (iter->value->listSize() && !isDerivation(name)) { - throw EvalError({ - .msg = hintfmt("tried to add derivation output context of %s, which is not a derivation, to a string", name), - .errPos = state.positions[i.pos] - }); + state.error( + "tried to add derivation output context of %s, which is not a derivation, to a string", + name + ).atPos(i.pos).debugThrow(); } for (auto elem : iter->value->listItems()) { auto outputName = state.forceStringNoCtx(*elem, iter->pos, "while evaluating an output name within a string context"); diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 27147a5d1..f51a6465d 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -23,20 +23,20 @@ static void runFetchClosureWithRewrite(EvalState & state, const PosIdx pos, Stor auto rewrittenPath = makeContentAddressed(fromStore, *state.store, fromPath); if (toPathMaybe && *toPathMaybe != rewrittenPath) throw Error({ - .msg = hintfmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected", + .msg = HintFmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected", state.store->printStorePath(fromPath), state.store->printStorePath(rewrittenPath), state.store->printStorePath(*toPathMaybe)), - .errPos = state.positions[pos] + .pos = state.positions[pos] }); if (!toPathMaybe) throw Error({ - .msg = hintfmt( + .msg = HintFmt( "rewriting '%s' to content-addressed form yielded '%s'\n" "Use this value for the 'toPath' attribute passed to 'fetchClosure'", state.store->printStorePath(fromPath), state.store->printStorePath(rewrittenPath)), - .errPos = state.positions[pos] + .pos = state.positions[pos] }); } @@ -50,11 +50,11 @@ static void runFetchClosureWithRewrite(EvalState & state, const PosIdx pos, Stor // We don't perform the rewriting when outPath already exists, as an optimisation. // However, we can quickly detect a mistake if the toPath is input addressed. throw Error({ - .msg = hintfmt( + .msg = HintFmt( "The 'toPath' value '%s' is input-addressed, so it can't possibly be the result of rewriting to a content-addressed path.\n\n" "Set 'toPath' to an empty string to make Nix report the correct content-addressed path.", state.store->printStorePath(toPath)), - .errPos = state.positions[pos] + .pos = state.positions[pos] }); } @@ -73,14 +73,14 @@ static void runFetchClosureWithContentAddressedPath(EvalState & state, const Pos if (!info->isContentAddressed(*state.store)) { throw Error({ - .msg = hintfmt( + .msg = HintFmt( "The 'fromPath' value '%s' is input-addressed, but 'inputAddressed' is set to 'false' (default).\n\n" "If you do intend to fetch an input-addressed store path, add\n\n" " inputAddressed = true;\n\n" "to the 'fetchClosure' arguments.\n\n" "Note that to ensure authenticity input-addressed store paths, users must configure a trusted binary cache public key on their systems. This is not needed for content-addressed paths.", state.store->printStorePath(fromPath)), - .errPos = state.positions[pos] + .pos = state.positions[pos] }); } @@ -99,11 +99,11 @@ static void runFetchClosureWithInputAddressedPath(EvalState & state, const PosId if (info->isContentAddressed(*state.store)) { throw Error({ - .msg = hintfmt( + .msg = HintFmt( "The store object referred to by 'fromPath' at '%s' is not input-addressed, but 'inputAddressed' is set to 'true'.\n\n" "Remove the 'inputAddressed' attribute (it defaults to 'false') to expect 'fromPath' to be content-addressed", state.store->printStorePath(fromPath)), - .errPos = state.positions[pos] + .pos = state.positions[pos] }); } @@ -153,15 +153,15 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg else throw Error({ - .msg = hintfmt("attribute '%s' isn't supported in call to 'fetchClosure'", attrName), - .errPos = state.positions[pos] + .msg = HintFmt("attribute '%s' isn't supported in call to 'fetchClosure'", attrName), + .pos = state.positions[pos] }); } if (!fromPath) throw Error({ - .msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "fromPath"), - .errPos = state.positions[pos] + .msg = HintFmt("attribute '%s' is missing in call to 'fetchClosure'", "fromPath"), + .pos = state.positions[pos] }); bool inputAddressed = inputAddressedMaybe.value_or(false); @@ -169,17 +169,17 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg if (inputAddressed) { if (toPath) throw Error({ - .msg = hintfmt("attribute '%s' is set to true, but '%s' is also set. Please remove one of them", + .msg = HintFmt("attribute '%s' is set to true, but '%s' is also set. Please remove one of them", "inputAddressed", "toPath"), - .errPos = state.positions[pos] + .pos = state.positions[pos] }); } if (!fromStoreUrl) throw Error({ - .msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "fromStore"), - .errPos = state.positions[pos] + .msg = HintFmt("attribute '%s' is missing in call to 'fetchClosure'", "fromStore"), + .pos = state.positions[pos] }); auto parsedURL = parseURL(*fromStoreUrl); @@ -188,14 +188,14 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg parsedURL.scheme != "https" && !(getEnv("_NIX_IN_TEST").has_value() && parsedURL.scheme == "file")) throw Error({ - .msg = hintfmt("'fetchClosure' only supports http:// and https:// stores"), - .errPos = state.positions[pos] + .msg = HintFmt("'fetchClosure' only supports http:// and https:// stores"), + .pos = state.positions[pos] }); if (!parsedURL.query.empty()) throw Error({ - .msg = hintfmt("'fetchClosure' does not support URL query parameters (in '%s')", *fromStoreUrl), - .errPos = state.positions[pos] + .msg = HintFmt("'fetchClosure' does not support URL query parameters (in '%s')", *fromStoreUrl), + .pos = state.positions[pos] }); auto fromStore = openStore(parsedURL.to_string()); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 58fe6f173..bb029b5b3 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -38,17 +38,11 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a else if (n == "name") name = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `name` attribute passed to builtins.fetchMercurial"); else - throw EvalError({ - .msg = hintfmt("unsupported argument '%s' to 'fetchMercurial'", state.symbols[attr.name]), - .errPos = state.positions[attr.pos] - }); + state.error("unsupported argument '%s' to 'fetchMercurial'", state.symbols[attr.name]).atPos(attr.pos).debugThrow(); } if (url.empty()) - throw EvalError({ - .msg = hintfmt("'url' argument required"), - .errPos = state.positions[pos] - }); + state.error("'url' argument required").atPos(pos).debugThrow(); } else url = state.coerceToString(pos, *args[0], context, diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index a943095bb..cfedfa6c4 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -9,6 +9,7 @@ #include "tarball.hh" #include "url.hh" #include "value-to-json.hh" +#include "fetch-to-store.hh" #include #include @@ -24,8 +25,6 @@ void emitTreeAttrs( bool emptyRevFallback, bool forceDirty) { - assert(input.isLocked()); - auto attrs = state.buildBindings(100); state.mkStorePathString(storePath, attrs.alloc(state.sOutPath)); @@ -100,16 +99,14 @@ static void fetchTree( if (auto aType = args[0]->attrs->get(state.sType)) { if (type) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("unexpected attribute 'type'"), - .errPos = state.positions[pos] - })); + state.error( + "unexpected attribute 'type'" + ).atPos(pos).debugThrow(); type = state.forceStringNoCtx(*aType->value, aType->pos, "while evaluating the `type` attribute passed to builtins.fetchTree"); } else if (!type) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("attribute 'type' is missing in call to 'fetchTree'"), - .errPos = state.positions[pos] - })); + state.error( + "attribute 'type' is missing in call to 'fetchTree'" + ).atPos(pos).debugThrow(); attrs.emplace("type", type.value()); @@ -132,8 +129,8 @@ static void fetchTree( attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, context).dump()); } else - state.debugThrowLastTrace(TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected", - state.symbols[attr.name], showType(*attr.value))); + state.error("fetchTree argument '%s' is %s while a string, Boolean or integer is expected", + state.symbols[attr.name], showType(*attr.value)).debugThrow(); } if (params.isFetchGit && !attrs.contains("exportIgnore") && (!attrs.contains("submodules") || !*fetchers::maybeGetBoolAttr(attrs, "submodules"))) { @@ -142,10 +139,9 @@ static void fetchTree( if (!params.allowNameArgument) if (auto nameIter = attrs.find("name"); nameIter != attrs.end()) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("attribute 'name' isn’t supported in call to 'fetchTree'"), - .errPos = state.positions[pos] - })); + state.error( + "attribute 'name' isn’t supported in call to 'fetchTree'" + ).atPos(pos).debugThrow(); input = fetchers::Input::fromAttrs(std::move(attrs)); } else { @@ -163,10 +159,9 @@ static void fetchTree( input = fetchers::Input::fromAttrs(std::move(attrs)); } else { if (!experimentalFeatureSettings.isEnabled(Xp::Flakes)) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("passing a string argument to 'fetchTree' requires the 'flakes' experimental feature"), - .errPos = state.positions[pos] - })); + state.error( + "passing a string argument to 'fetchTree' requires the 'flakes' experimental feature" + ).atPos(pos).debugThrow(); input = fetchers::Input::fromURL(url); } } @@ -175,10 +170,14 @@ static void fetchTree( input = lookupInRegistries(state.store, input).first; if (evalSettings.pureEval && !input.isLocked()) { + auto fetcher = "fetchTree"; if (params.isFetchGit) - state.debugThrowLastTrace(EvalError("in pure evaluation mode, 'fetchGit' requires a locked input, at %s", state.positions[pos])); - else - state.debugThrowLastTrace(EvalError("in pure evaluation mode, 'fetchTree' requires a locked input, at %s", state.positions[pos])); + fetcher = "fetchGit"; + + state.error( + "in pure evaluation mode, '%s' will not fetch unlocked input '%s'", + fetcher, input.to_string() + ).atPos(pos).debugThrow(); } state.checkURI(input.toURLString()); @@ -432,17 +431,13 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v else if (n == "name") name = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the name of the content we should fetch"); else - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("unsupported argument '%s' to '%s'", n, who), - .errPos = state.positions[attr.pos] - })); + state.error("unsupported argument '%s' to '%s'", n, who) + .atPos(pos).debugThrow(); } if (!url) - state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("'url' argument required"), - .errPos = state.positions[pos] - })); + state.error( + "'url' argument required").atPos(pos).debugThrow(); } else url = state.forceStringNoCtx(*args[0], pos, "while evaluating the url we should fetch"); @@ -455,7 +450,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v name = baseNameOf(*url); if (evalSettings.pureEval && !expectedHash) - state.debugThrowLastTrace(EvalError("in pure evaluation mode, '%s' requires a 'sha256' argument", who)); + state.error("in pure evaluation mode, '%s' requires a 'sha256' argument", who).atPos(pos).debugThrow(); // early exit if pinned and already in the store if (expectedHash && expectedHash->algo == HashAlgorithm::SHA256) { @@ -477,16 +472,22 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v // https://github.com/NixOS/nix/issues/4313 auto storePath = unpack - ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath + ? fetchToStore(*state.store, fetchers::downloadTarball(*url).accessor, FetchMode::Copy, name) : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath; if (expectedHash) { auto hash = unpack ? state.store->queryPathInfo(storePath)->narHash : hashFile(HashAlgorithm::SHA256, state.store->toRealPath(storePath)); - if (hash != *expectedHash) - state.debugThrowLastTrace(EvalError((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s", - *url, expectedHash->to_string(HashFormat::Nix32, true), hash.to_string(HashFormat::Nix32, true))); + if (hash != *expectedHash) { + state.error( + "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s", + *url, + expectedHash->to_string(HashFormat::Nix32, true), + hash.to_string(HashFormat::Nix32, true) + ).withExitStatus(102) + .debugThrow(); + } } state.allowAndSetStorePathString(storePath, v); diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 2f4d4022e..94be7960a 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -83,10 +83,7 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V try { visit(val, toml::parse(tomlStream, "fromTOML" /* the "filename" */)); } catch (std::exception & e) { // TODO: toml::syntax_error - throw EvalError({ - .msg = hintfmt("while parsing a TOML string: %s", e.what()), - .errPos = state.positions[pos] - }); + state.error("while parsing TOML: %s", e.what()).atPos(pos).debugThrow(); } } diff --git a/src/libexpr/print-options.hh b/src/libexpr/print-options.hh index e03746ece..6c5e80c61 100644 --- a/src/libexpr/print-options.hh +++ b/src/libexpr/print-options.hh @@ -17,24 +17,29 @@ struct PrintOptions * If true, output ANSI color sequences. */ bool ansiColors = false; + /** * If true, force values. */ bool force = false; + /** * If true and `force` is set, print derivations as * `«derivation /nix/store/...»` instead of as attribute sets. */ bool derivationPaths = false; + /** * If true, track which values have been printed and skip them on * subsequent encounters. Useful for self-referential values. */ bool trackRepeated = true; + /** * Maximum depth to evaluate to. */ size_t maxDepth = std::numeric_limits::max(); + /** * Maximum number of attributes in attribute sets to print. * @@ -42,6 +47,7 @@ struct PrintOptions * attribute set encountered. */ size_t maxAttrs = std::numeric_limits::max(); + /** * Maximum number of list items to print. * @@ -49,10 +55,26 @@ struct PrintOptions * list encountered. */ size_t maxListItems = std::numeric_limits::max(); + /** * Maximum string length to print. */ size_t maxStringLength = std::numeric_limits::max(); + + /** + * Indentation width for pretty-printing. + * + * If set to 0 (the default), values are not pretty-printed. + */ + size_t prettyIndent = 0; + + /** + * True if pretty-printing is enabled. + */ + inline bool shouldPrettyPrint() + { + return prettyIndent > 0; + } }; /** diff --git a/src/libexpr/print.cc b/src/libexpr/print.cc index 702e4bfe8..9d280f623 100644 --- a/src/libexpr/print.cc +++ b/src/libexpr/print.cc @@ -152,7 +152,8 @@ struct ImportantFirstAttrNameCmp } }; -typedef std::set ValuesSeen; +typedef std::set ValuesSeen; +typedef std::vector> AttrVec; class Printer { @@ -163,6 +164,37 @@ private: std::optional seen; size_t attrsPrinted = 0; size_t listItemsPrinted = 0; + std::string indent; + + void increaseIndent() + { + if (options.shouldPrettyPrint()) { + indent.append(options.prettyIndent, ' '); + } + } + + void decreaseIndent() + { + if (options.shouldPrettyPrint()) { + assert(indent.size() >= options.prettyIndent); + indent.resize(indent.size() - options.prettyIndent); + } + } + + /** + * Print a space (for separating items or attributes). + * + * If pretty-printing is enabled, a newline and the current `indent` is + * printed instead. + */ + void printSpace(bool prettyPrint) + { + if (prettyPrint) { + output << "\n" << indent; + } else { + output << " "; + } + } void printRepeated() { @@ -255,14 +287,36 @@ private: output << "»"; if (options.ansiColors) output << ANSI_NORMAL; - } catch (BaseError & e) { + } catch (Error & e) { printError_(e); } } + bool shouldPrettyPrintAttrs(AttrVec & v) + { + if (!options.shouldPrettyPrint() || v.empty()) { + return false; + } + + // Pretty-print attrsets with more than one item. + if (v.size() > 1) { + return true; + } + + auto item = v[0].second; + if (!item) { + return true; + } + + // Pretty-print single-item attrsets only if they contain nested + // structures. + auto itemType = item->type(); + return itemType == nList || itemType == nAttrs || itemType == nThunk; + } + void printAttrs(Value & v, size_t depth) { - if (seen && !seen->insert(&v).second) { + if (seen && !seen->insert(v.attrs).second) { printRepeated(); return; } @@ -270,9 +324,10 @@ private: if (options.force && options.derivationPaths && state.isDerivation(v)) { printDerivation(v); } else if (depth < options.maxDepth) { - output << "{ "; + increaseIndent(); + output << "{"; - std::vector> sorted; + AttrVec sorted; for (auto & i : *v.attrs) sorted.emplace_back(std::pair(state.symbols[i.name], i.value)); @@ -281,7 +336,11 @@ private: else std::sort(sorted.begin(), sorted.end(), ImportantFirstAttrNameCmp()); + auto prettyPrint = shouldPrettyPrintAttrs(sorted); + for (auto & i : sorted) { + printSpace(prettyPrint); + if (attrsPrinted >= options.maxAttrs) { printElided(sorted.size() - attrsPrinted, "attribute", "attributes"); break; @@ -290,13 +349,38 @@ private: printAttributeName(output, i.first); output << " = "; print(*i.second, depth + 1); - output << "; "; + output << ";"; attrsPrinted++; } + decreaseIndent(); + printSpace(prettyPrint); output << "}"; - } else + } else { output << "{ ... }"; + } + } + + bool shouldPrettyPrintList(std::span list) + { + if (!options.shouldPrettyPrint() || list.empty()) { + return false; + } + + // Pretty-print lists with more than one item. + if (list.size() > 1) { + return true; + } + + auto item = list[0]; + if (!item) { + return true; + } + + // Pretty-print single-item lists only if they contain nested + // structures. + auto itemType = item->type(); + return itemType == nList || itemType == nAttrs || itemType == nThunk; } void printList(Value & v, size_t depth) @@ -306,11 +390,16 @@ private: return; } - output << "[ "; if (depth < options.maxDepth) { - for (auto elem : v.listItems()) { + increaseIndent(); + output << "["; + auto listItems = v.listItems(); + auto prettyPrint = shouldPrettyPrintList(listItems); + for (auto elem : listItems) { + printSpace(prettyPrint); + if (listItemsPrinted >= options.maxListItems) { - printElided(v.listSize() - listItemsPrinted, "item", "items"); + printElided(listItems.size() - listItemsPrinted, "item", "items"); break; } @@ -319,13 +408,15 @@ private: } else { printNullptr(); } - output << " "; listItemsPrinted++; } + + decreaseIndent(); + printSpace(prettyPrint); + output << "]"; + } else { + output << "[ ... ]"; } - else - output << "... "; - output << "]"; } void printFunction(Value & v) @@ -405,11 +496,11 @@ private: output << ANSI_NORMAL; } - void printError_(BaseError & e) + void printError_(Error & e) { if (options.ansiColors) output << ANSI_RED; - output << "«" << e.msg() << "»"; + output << "«error: " << filterANSIEscapes(e.info().msg.str(), true) << "»"; if (options.ansiColors) output << ANSI_NORMAL; } @@ -422,7 +513,7 @@ private: if (options.force) { try { state.forceValue(v, v.determinePos(noPos)); - } catch (BaseError & e) { + } catch (Error & e) { printError_(e); return; } @@ -488,6 +579,7 @@ public: { attrsPrinted = 0; listItemsPrinted = 0; + indent.clear(); if (options.trackRepeated) { seen.emplace(); @@ -511,4 +603,11 @@ std::ostream & operator<<(std::ostream & output, const ValuePrinter & printer) return output; } +template<> +HintFmt & HintFmt::operator%(const ValuePrinter & value) +{ + fmt % value; + return *this; +} + } diff --git a/src/libexpr/print.hh b/src/libexpr/print.hh index a8300264a..7ddda81b8 100644 --- a/src/libexpr/print.hh +++ b/src/libexpr/print.hh @@ -9,6 +9,7 @@ #include +#include "fmt.hh" #include "print-options.hh" namespace nix { @@ -78,4 +79,13 @@ public: }; std::ostream & operator<<(std::ostream & output, const ValuePrinter & printer); + + +/** + * `ValuePrinter` does its own ANSI formatting, so we don't color it + * magenta. + */ +template<> +HintFmt & HintFmt::operator%(const ValuePrinter & value); + } diff --git a/src/libexpr/repl-exit-status.hh b/src/libexpr/repl-exit-status.hh new file mode 100644 index 000000000..08299ff61 --- /dev/null +++ b/src/libexpr/repl-exit-status.hh @@ -0,0 +1,20 @@ +#pragma once + +namespace nix { + +/** + * Exit status returned from the REPL. + */ +enum class ReplExitStatus { + /** + * The user exited with `:quit`. The program (e.g., if the REPL was acting + * as the debugger) should exit. + */ + QuitAll, + /** + * The user exited with `:continue`. The program should continue running. + */ + Continue, +}; + +} diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 74b3ebf13..3f877a7fd 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -64,7 +64,7 @@ json printValueAsJSON(EvalState & state, bool strict, out[j] = printValueAsJSON(state, strict, *a.value, a.pos, context, copyToStore); } catch (Error & e) { e.addTrace(state.positions[a.pos], - hintfmt("while evaluating attribute '%1%'", j)); + HintFmt("while evaluating attribute '%1%'", j)); throw; } } @@ -80,8 +80,8 @@ json printValueAsJSON(EvalState & state, bool strict, try { out.push_back(printValueAsJSON(state, strict, *elem, pos, context, copyToStore)); } catch (Error & e) { - e.addTrace({}, - hintfmt("while evaluating list element at index %1%", i)); + e.addTrace(state.positions[pos], + HintFmt("while evaluating list element at index %1%", i)); throw; } i++; @@ -99,13 +99,12 @@ json printValueAsJSON(EvalState & state, bool strict, case nThunk: case nFunction: - auto e = TypeError({ - .msg = hintfmt("cannot convert %1% to JSON", showType(v)), - .errPos = state.positions[v.determinePos(pos)] - }); - e.addTrace(state.positions[pos], hintfmt("message for the trace")); - state.debugThrowLastTrace(e); - throw e; + state.error( + "cannot convert %1% to JSON", + showType(v) + ) + .atPos(v.determinePos(pos)) + .debugThrow(); } return out; } @@ -119,7 +118,8 @@ void printValueAsJSON(EvalState & state, bool strict, json ExternalValueBase::printValueAsJSON(EvalState & state, bool strict, NixStringContext & context, bool copyToStore) const { - state.debugThrowLastTrace(TypeError("cannot convert %1% to JSON", showType())); + state.error("cannot convert %1% to JSON", showType()) + .debugThrow(); } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 214d52271..e7aea4949 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -105,7 +105,7 @@ class ExternalValueBase * Coerce the value to a string. Defaults to uncoercable, i.e. throws an * error. */ - virtual std::string coerceToString(const Pos & pos, NixStringContext & context, bool copyMore, bool copyToStore) const; + virtual std::string coerceToString(EvalState & state, const PosIdx & pos, NixStringContext & context, bool copyMore, bool copyToStore) const; /** * Compare to another value of the same type. Defaults to uncomparable, diff --git a/src/libexpr/value/context.hh b/src/libexpr/value/context.hh index 51fd30a44..7f23cd3a4 100644 --- a/src/libexpr/value/context.hh +++ b/src/libexpr/value/context.hh @@ -19,8 +19,8 @@ public: : Error("") { raw = raw_; - auto hf = hintfmt(args...); - err.msg = hintfmt("Bad String Context element: %1%: %2%", normaltxt(hf.str()), raw); + auto hf = HintFmt(args...); + err.msg = HintFmt("Bad String Context element: %1%: %2%", Uncolored(hf.str()), raw); } }; diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index a565d19d4..b788c5948 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -104,4 +104,9 @@ std::map attrsToQuery(const Attrs & attrs) return query; } +Hash getRevAttr(const Attrs & attrs, const std::string & name) +{ + return Hash::parseAny(getStrAttr(attrs, name), HashAlgorithm::SHA1); +} + } diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh index b9a2c824e..97a74bce0 100644 --- a/src/libfetchers/attrs.hh +++ b/src/libfetchers/attrs.hh @@ -39,4 +39,6 @@ bool getBoolAttr(const Attrs & attrs, const std::string & name); std::map attrsToQuery(const Attrs & attrs); +Hash getRevAttr(const Attrs & attrs, const std::string & name); + } diff --git a/src/libfetchers/fetch-to-store.cc b/src/libfetchers/fetch-to-store.cc index 196489e05..398286065 100644 --- a/src/libfetchers/fetch-to-store.cc +++ b/src/libfetchers/fetch-to-store.cc @@ -7,6 +7,7 @@ namespace nix { StorePath fetchToStore( Store & store, const SourcePath & path, + FetchMode mode, std::string_view name, ContentAddressMethod method, PathFilter * filter, @@ -21,23 +22,9 @@ StorePath fetchToStore( cacheKey = fetchers::Attrs{ {"_what", "fetchToStore"}, {"store", store.storeDir}, - {"name", std::string(name)}, + {"name", std::string{name}}, {"fingerprint", *path.accessor->fingerprint}, - { - "method", - std::visit(overloaded { - [](const TextIngestionMethod &) { - return "text"; - }, - [](const FileIngestionMethod & fim) { - switch (fim) { - case FileIngestionMethod::Flat: return "flat"; - case FileIngestionMethod::Recursive: return "nar"; - default: assert(false); - } - }, - }, method.raw), - }, + {"method", std::string{method.render()}}, {"path", path.path.abs()} }; if (auto res = fetchers::getCache()->lookup(store, *cacheKey)) { @@ -47,18 +34,19 @@ StorePath fetchToStore( } else debug("source path '%s' is uncacheable", path); - Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", path)); + Activity act(*logger, lvlChatty, actUnknown, + fmt(mode == FetchMode::DryRun ? "hashing '%s'" : "copying '%s' to the store", path)); auto filter2 = filter ? *filter : defaultPathFilter; auto storePath = - settings.readOnlyMode + mode == FetchMode::DryRun ? store.computeStorePath( name, *path.accessor, path.path, method, HashAlgorithm::SHA256, {}, filter2).first : store.addToStore( name, *path.accessor, path.path, method, HashAlgorithm::SHA256, {}, filter2, repair); - if (cacheKey) + if (cacheKey && mode == FetchMode::Copy) fetchers::getCache()->add(store, *cacheKey, {}, storePath, true); return storePath; diff --git a/src/libfetchers/fetch-to-store.hh b/src/libfetchers/fetch-to-store.hh index e5e039340..81af1e240 100644 --- a/src/libfetchers/fetch-to-store.hh +++ b/src/libfetchers/fetch-to-store.hh @@ -8,12 +8,15 @@ namespace nix { +enum struct FetchMode { DryRun, Copy }; + /** * Copy the `path` to the Nix store. */ StorePath fetchToStore( Store & store, const SourcePath & path, + FetchMode mode, std::string_view name = "source", ContentAddressMethod method = FileIngestionMethod::Recursive, PathFilter * filter = nullptr, diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 7f282c972..363ad018e 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -45,12 +45,8 @@ static void fixupInput(Input & input) // Check common attributes. input.getType(); input.getRef(); - if (input.getRev()) - input.locked = true; input.getRevCount(); input.getLastModified(); - if (input.getNarHash()) - input.locked = true; } Input Input::fromURL(const ParsedURL & url, bool requireTree) @@ -140,6 +136,11 @@ bool Input::isDirect() const return !scheme || scheme->isDirect(*this); } +bool Input::isLocked() const +{ + return scheme && scheme->isLocked(*this); +} + Attrs Input::toAttrs() const { return attrs; @@ -222,8 +223,6 @@ std::pair Input::fetch(ref store) const input.to_string(), *prevRevCount); } - input.locked = true; - return {std::move(storePath), input}; } @@ -376,7 +375,7 @@ void InputScheme::clone(const Input & input, const Path & destDir) const std::pair InputScheme::fetch(ref store, const Input & input) { auto [accessor, input2] = getAccessor(store, input); - auto storePath = fetchToStore(*store, SourcePath(accessor), input2.getName()); + auto storePath = fetchToStore(*store, SourcePath(accessor), FetchMode::Copy, input2.getName()); return {storePath, input2}; } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 036647830..472fba6f4 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -29,7 +29,6 @@ struct Input std::shared_ptr scheme; // note: can be null Attrs attrs; - bool locked = false; /** * path of the parent of this input, used for relative path resolution @@ -71,7 +70,7 @@ public: * Check whether this is a "locked" input, that is, * one that contains a commit hash or content hash. */ - bool isLocked() const { return locked; } + bool isLocked() const; bool operator ==(const Input & other) const; @@ -121,7 +120,6 @@ public: std::optional getFingerprint(ref store) const; }; - /** * The `InputScheme` represents a type of fetcher. Each fetcher * registers with nix at startup time. When processing an `Input`, @@ -196,6 +194,14 @@ struct InputScheme */ virtual std::optional getFingerprint(ref store, const Input & input) const { return std::nullopt; } + + /** + * Return `true` if this input is considered "locked", i.e. it has + * attributes like a Git revision or NAR hash that uniquely + * identify its contents. + */ + virtual bool isLocked(const Input & input) const + { return false; } }; void registerInputScheme(std::shared_ptr && fetcher); diff --git a/src/libfetchers/filtering-input-accessor.cc b/src/libfetchers/filtering-input-accessor.cc index 581ce3c1d..32343abc4 100644 --- a/src/libfetchers/filtering-input-accessor.cc +++ b/src/libfetchers/filtering-input-accessor.cc @@ -5,26 +5,26 @@ namespace nix { std::string FilteringInputAccessor::readFile(const CanonPath & path) { checkAccess(path); - return next->readFile(prefix + path); + return next->readFile(prefix / path); } bool FilteringInputAccessor::pathExists(const CanonPath & path) { - return isAllowed(path) && next->pathExists(prefix + path); + return isAllowed(path) && next->pathExists(prefix / path); } std::optional FilteringInputAccessor::maybeLstat(const CanonPath & path) { checkAccess(path); - return next->maybeLstat(prefix + path); + return next->maybeLstat(prefix / path); } InputAccessor::DirEntries FilteringInputAccessor::readDirectory(const CanonPath & path) { checkAccess(path); DirEntries entries; - for (auto & entry : next->readDirectory(prefix + path)) { - if (isAllowed(path + entry.first)) + for (auto & entry : next->readDirectory(prefix / path)) { + if (isAllowed(path / entry.first)) entries.insert(std::move(entry)); } return entries; @@ -33,12 +33,12 @@ InputAccessor::DirEntries FilteringInputAccessor::readDirectory(const CanonPath std::string FilteringInputAccessor::readLink(const CanonPath & path) { checkAccess(path); - return next->readLink(prefix + path); + return next->readLink(prefix / path); } std::string FilteringInputAccessor::showPath(const CanonPath & path) { - return next->showPath(prefix + path); + return next->showPath(prefix / path); } void FilteringInputAccessor::checkAccess(const CanonPath & path) @@ -51,33 +51,33 @@ void FilteringInputAccessor::checkAccess(const CanonPath & path) struct AllowListInputAccessorImpl : AllowListInputAccessor { - std::set allowedPaths; + std::set allowedPrefixes; AllowListInputAccessorImpl( ref next, - std::set && allowedPaths, + std::set && allowedPrefixes, MakeNotAllowedError && makeNotAllowedError) : AllowListInputAccessor(SourcePath(next), std::move(makeNotAllowedError)) - , allowedPaths(std::move(allowedPaths)) + , allowedPrefixes(std::move(allowedPrefixes)) { } bool isAllowed(const CanonPath & path) override { - return path.isAllowed(allowedPaths); + return path.isAllowed(allowedPrefixes); } - void allowPath(CanonPath path) override + void allowPrefix(CanonPath prefix) override { - allowedPaths.insert(std::move(path)); + allowedPrefixes.insert(std::move(prefix)); } }; ref AllowListInputAccessor::create( ref next, - std::set && allowedPaths, + std::set && allowedPrefixes, MakeNotAllowedError && makeNotAllowedError) { - return make_ref(next, std::move(allowedPaths), std::move(makeNotAllowedError)); + return make_ref(next, std::move(allowedPrefixes), std::move(makeNotAllowedError)); } bool CachingFilteringInputAccessor::isAllowed(const CanonPath & path) diff --git a/src/libfetchers/filtering-input-accessor.hh b/src/libfetchers/filtering-input-accessor.hh index 8a9b206ee..8111a72c5 100644 --- a/src/libfetchers/filtering-input-accessor.hh +++ b/src/libfetchers/filtering-input-accessor.hh @@ -54,18 +54,19 @@ struct FilteringInputAccessor : InputAccessor }; /** - * A wrapping `InputAccessor` that checks paths against an allow-list. + * A wrapping `InputAccessor` that checks paths against a set of + * allowed prefixes. */ struct AllowListInputAccessor : public FilteringInputAccessor { /** - * Grant access to the specified path. + * Grant access to the specified prefix. */ - virtual void allowPath(CanonPath path) = 0; + virtual void allowPrefix(CanonPath prefix) = 0; static ref create( ref next, - std::set && allowedPaths, + std::set && allowedPrefixes, MakeNotAllowedError && makeNotAllowedError); using FilteringInputAccessor::FilteringInputAccessor; diff --git a/src/libfetchers/fs-input-accessor.cc b/src/libfetchers/fs-input-accessor.cc index c3d8d273c..ee24c621a 100644 --- a/src/libfetchers/fs-input-accessor.cc +++ b/src/libfetchers/fs-input-accessor.cc @@ -6,72 +6,30 @@ namespace nix { struct FSInputAccessor : InputAccessor, PosixSourceAccessor { - CanonPath root; - - FSInputAccessor(const CanonPath & root) - : root(root) - { - displayPrefix = root.isRoot() ? "" : root.abs(); - } - - void readFile( - const CanonPath & path, - Sink & sink, - std::function sizeCallback) override - { - auto absPath = makeAbsPath(path); - PosixSourceAccessor::readFile(absPath, sink, sizeCallback); - } - - bool pathExists(const CanonPath & path) override - { - return PosixSourceAccessor::pathExists(makeAbsPath(path)); - } - - std::optional maybeLstat(const CanonPath & path) override - { - return PosixSourceAccessor::maybeLstat(makeAbsPath(path)); - } - - DirEntries readDirectory(const CanonPath & path) override - { - DirEntries res; - for (auto & entry : PosixSourceAccessor::readDirectory(makeAbsPath(path))) - res.emplace(entry); - return res; - } - - std::string readLink(const CanonPath & path) override - { - return PosixSourceAccessor::readLink(makeAbsPath(path)); - } - - CanonPath makeAbsPath(const CanonPath & path) - { - return root + path; - } - - std::optional getPhysicalPath(const CanonPath & path) override - { - return makeAbsPath(path); - } + using PosixSourceAccessor::PosixSourceAccessor; }; -ref makeFSInputAccessor(const CanonPath & root) +ref makeFSInputAccessor() { - return make_ref(root); + return make_ref(); +} + +ref makeFSInputAccessor(std::filesystem::path root) +{ + return make_ref(std::move(root)); } ref makeStorePathAccessor( ref store, const StorePath & storePath) { - return makeFSInputAccessor(CanonPath(store->toRealPath(storePath))); + // FIXME: should use `store->getFSAccessor()` + return makeFSInputAccessor(std::filesystem::path { store->toRealPath(storePath) }); } SourcePath getUnfilteredRootPath(CanonPath path) { - static auto rootFS = makeFSInputAccessor(CanonPath::root); + static auto rootFS = makeFSInputAccessor(); return {rootFS, path}; } diff --git a/src/libfetchers/fs-input-accessor.hh b/src/libfetchers/fs-input-accessor.hh index a98e83511..e60906bd8 100644 --- a/src/libfetchers/fs-input-accessor.hh +++ b/src/libfetchers/fs-input-accessor.hh @@ -8,8 +8,9 @@ namespace nix { class StorePath; class Store; -ref makeFSInputAccessor( - const CanonPath & root); +ref makeFSInputAccessor(); + +ref makeFSInputAccessor(std::filesystem::path root); ref makeStorePathAccessor( ref store, diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 382a363f0..9cae9034e 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -2,12 +2,13 @@ #include "fs-input-accessor.hh" #include "input-accessor.hh" #include "filtering-input-accessor.hh" +#include "memory-input-accessor.hh" #include "cache.hh" #include "finally.hh" #include "processes.hh" #include "signals.hh" - -#include +#include "users.hh" +#include "fs-sink.hh" #include #include @@ -28,6 +29,7 @@ #include #include #include +#include namespace std { @@ -140,15 +142,15 @@ T peelObject(git_repository * repo, git_object * obj, git_object_t type) struct GitRepoImpl : GitRepo, std::enable_shared_from_this { /** Location of the repository on disk. */ - CanonPath path; + std::filesystem::path path; Repository repo; - GitRepoImpl(CanonPath _path, bool create, bool bare) + GitRepoImpl(std::filesystem::path _path, bool create, bool bare) : path(std::move(_path)) { initLibGit2(); - if (pathExists(path.abs())) { + if (pathExists(path.native())) { if (git_repository_open(Setter(repo), path.c_str())) throw Error("opening Git repository '%s': %s", path, git_error_last()->message); } else { @@ -221,10 +223,10 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this return toHash(*oid); } - std::vector parseSubmodules(const CanonPath & configFile) + std::vector parseSubmodules(const std::filesystem::path & configFile) { GitConfig config; - if (git_config_open_ondisk(Setter(config), configFile.abs().c_str())) + if (git_config_open_ondisk(Setter(config), configFile.c_str())) throw Error("parsing .gitmodules file: %s", git_error_last()->message); ConfigIterator it; @@ -295,8 +297,8 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this throw Error("getting working directory status: %s", git_error_last()->message); /* Get submodule info. */ - auto modulesFile = path + ".gitmodules"; - if (pathExists(modulesFile.abs())) + auto modulesFile = path / ".gitmodules"; + if (pathExists(modulesFile)) info.submodules = parseSubmodules(modulesFile); return info; @@ -356,6 +358,8 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this ref getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError e) override; + ref getFileSystemObjectSink() override; + static int sidebandProgressCallback(const char * str, int len, void * payload) { auto act = (Activity *) payload; @@ -389,10 +393,10 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this auto dir = this->path; Strings gitArgs; if (shallow) { - gitArgs = { "-C", dir.abs(), "fetch", "--quiet", "--force", "--depth", "1", "--", url, refspec }; + gitArgs = { "-C", dir, "fetch", "--quiet", "--force", "--depth", "1", "--", url, refspec }; } else { - gitArgs = { "-C", dir.abs(), "fetch", "--quiet", "--force", "--", url, refspec }; + gitArgs = { "-C", dir, "fetch", "--quiet", "--force", "--", url, refspec }; } runProgram(RunOptions { @@ -438,7 +442,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this .args = { "-c", "gpg.ssh.allowedSignersFile=" + allowedSignersFile, - "-C", path.abs(), + "-C", path, "verify-commit", rev.gitRev() }, @@ -463,9 +467,25 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this else throw Error("Commit signature verification on commit %s failed: %s", rev.gitRev(), output); } + + Hash treeHashToNarHash(const Hash & treeHash) override + { + auto accessor = getAccessor(treeHash, false); + + fetchers::Attrs cacheKey({{"_what", "treeHashToNarHash"}, {"treeHash", treeHash.gitRev()}}); + + if (auto res = fetchers::getCache()->lookup(cacheKey)) + return Hash::parseAny(fetchers::getStrAttr(*res, "narHash"), HashAlgorithm::SHA256); + + auto narHash = accessor->hashPath(CanonPath::root); + + fetchers::getCache()->upsert(cacheKey, fetchers::Attrs({{"narHash", narHash.to_string(HashFormat::SRI, true)}})); + + return narHash; + } }; -ref GitRepo::openRepo(const CanonPath & path, bool create, bool bare) +ref GitRepo::openRepo(const std::filesystem::path & path, bool create, bool bare) { return make_ref(path, create, bare); } @@ -576,20 +596,61 @@ struct GitInputAccessor : InputAccessor /* Recursively look up 'path' relative to the root. */ git_tree_entry * lookup(const CanonPath & path) { - if (path.isRoot()) return nullptr; - auto i = lookupCache.find(path); - if (i == lookupCache.end()) { - TreeEntry entry; - if (auto err = git_tree_entry_bypath(Setter(entry), root.get(), std::string(path.rel()).c_str())) { - if (err != GIT_ENOTFOUND) - throw Error("looking up '%s': %s", showPath(path), git_error_last()->message); - } + if (i != lookupCache.end()) return i->second.get(); - i = lookupCache.emplace(path, std::move(entry)).first; + auto parent = path.parent(); + if (!parent) return nullptr; + + auto name = path.baseName().value(); + + auto parentTree = lookupTree(*parent); + if (!parentTree) return nullptr; + + auto count = git_tree_entrycount(parentTree->get()); + + git_tree_entry * res = nullptr; + + /* Add all the tree entries to the cache to speed up + subsequent lookups. */ + for (size_t n = 0; n < count; ++n) { + auto entry = git_tree_entry_byindex(parentTree->get(), n); + + TreeEntry copy; + if (git_tree_entry_dup(Setter(copy), entry)) + throw Error("dupping tree entry: %s", git_error_last()->message); + + auto entryName = std::string_view(git_tree_entry_name(entry)); + + if (entryName == name) + res = copy.get(); + + auto path2 = *parent; + path2.push(entryName); + lookupCache.emplace(path2, std::move(copy)).first->second.get(); } - return &*i->second; + return res; + } + + std::optional lookupTree(const CanonPath & path) + { + if (path.isRoot()) { + Tree tree; + if (git_tree_dup(Setter(tree), root.get())) + throw Error("duplicating directory '%s': %s", showPath(path), git_error_last()->message); + return tree; + } + + auto entry = lookup(path); + if (!entry || git_tree_entry_type(entry) != GIT_OBJECT_TREE) + return std::nullopt; + + Tree tree; + if (git_tree_entry_to_object((git_object * *) (git_tree * *) Setter(tree), *repo, entry)) + throw Error("looking up directory '%s': %s", showPath(path), git_error_last()->message); + + return tree; } git_tree_entry * need(const CanonPath & path) @@ -729,6 +790,154 @@ struct GitExportIgnoreInputAccessor : CachingFilteringInputAccessor { }; +struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink +{ + ref repo; + + struct PendingDir + { + std::string name; + TreeBuilder builder; + }; + + std::vector pendingDirs; + + size_t componentsToStrip = 1; + + void pushBuilder(std::string name) + { + git_treebuilder * b; + if (git_treebuilder_new(&b, *repo, nullptr)) + throw Error("creating a tree builder: %s", git_error_last()->message); + pendingDirs.push_back({ .name = std::move(name), .builder = TreeBuilder(b) }); + }; + + GitFileSystemObjectSinkImpl(ref repo) : repo(repo) + { + pushBuilder(""); + } + + std::pair popBuilder() + { + assert(!pendingDirs.empty()); + auto pending = std::move(pendingDirs.back()); + git_oid oid; + if (git_treebuilder_write(&oid, pending.builder.get())) + throw Error("creating a tree object: %s", git_error_last()->message); + pendingDirs.pop_back(); + return {oid, pending.name}; + }; + + void addToTree(const std::string & name, const git_oid & oid, git_filemode_t mode) + { + assert(!pendingDirs.empty()); + auto & pending = pendingDirs.back(); + if (git_treebuilder_insert(nullptr, pending.builder.get(), name.c_str(), &oid, mode)) + throw Error("adding a file to a tree builder: %s", git_error_last()->message); + }; + + void updateBuilders(std::span names) + { + // Find the common prefix of pendingDirs and names. + size_t prefixLen = 0; + for (; prefixLen < names.size() && prefixLen + 1 < pendingDirs.size(); ++prefixLen) + if (names[prefixLen] != pendingDirs[prefixLen + 1].name) + break; + + // Finish the builders that are not part of the common prefix. + for (auto n = pendingDirs.size(); n > prefixLen + 1; --n) { + auto [oid, name] = popBuilder(); + addToTree(name, oid, GIT_FILEMODE_TREE); + } + + // Create builders for the new directories. + for (auto n = prefixLen; n < names.size(); ++n) + pushBuilder(names[n]); + }; + + bool prepareDirs(const std::vector & pathComponents, bool isDir) + { + std::span pathComponents2{pathComponents}; + + if (pathComponents2.size() <= componentsToStrip) return false; + pathComponents2 = pathComponents2.subspan(componentsToStrip); + + updateBuilders( + isDir + ? pathComponents2 + : pathComponents2.first(pathComponents2.size() - 1)); + + return true; + } + + void createRegularFile( + const Path & path, + std::function func) override + { + auto pathComponents = tokenizeString>(path, "/"); + if (!prepareDirs(pathComponents, false)) return; + + git_writestream * stream = nullptr; + if (git_blob_create_from_stream(&stream, *repo, nullptr)) + throw Error("creating a blob stream object: %s", git_error_last()->message); + + struct CRF : CreateRegularFileSink { + const Path & path; + GitFileSystemObjectSinkImpl & back; + git_writestream * stream; + bool executable = false; + CRF(const Path & path, GitFileSystemObjectSinkImpl & back, git_writestream * stream) + : path(path), back(back), stream(stream) + {} + void operator () (std::string_view data) override + { + if (stream->write(stream, data.data(), data.size())) + throw Error("writing a blob for tarball member '%s': %s", path, git_error_last()->message); + } + void isExecutable() override + { + executable = true; + } + } crf { path, *this, stream }; + func(crf); + + git_oid oid; + if (git_blob_create_from_stream_commit(&oid, stream)) + throw Error("creating a blob object for tarball member '%s': %s", path, git_error_last()->message); + + addToTree(*pathComponents.rbegin(), oid, + crf.executable + ? GIT_FILEMODE_BLOB_EXECUTABLE + : GIT_FILEMODE_BLOB); + } + + void createDirectory(const Path & path) override + { + auto pathComponents = tokenizeString>(path, "/"); + (void) prepareDirs(pathComponents, true); + } + + void createSymlink(const Path & path, const std::string & target) override + { + auto pathComponents = tokenizeString>(path, "/"); + if (!prepareDirs(pathComponents, false)) return; + + git_oid oid; + if (git_blob_create_from_buffer(&oid, *repo, target.c_str(), target.size())) + throw Error("creating a blob object for tarball symlink member '%s': %s", path, git_error_last()->message); + + addToTree(*pathComponents.rbegin(), oid, GIT_FILEMODE_LINK); + } + + Hash sync() override { + updateBuilders({}); + + auto [oid, _name] = popBuilder(); + + return toHash(oid); + } +}; + ref GitRepoImpl::getRawAccessor(const Hash & rev) { auto self = ref(shared_from_this()); @@ -750,17 +959,26 @@ ref GitRepoImpl::getAccessor(const Hash & rev, bool exportIgnore) ref GitRepoImpl::getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError) { auto self = ref(shared_from_this()); + /* In case of an empty workdir, return an empty in-memory tree. We + cannot use AllowListInputAccessor because it would return an + error for the root (and we can't add the root to the allow-list + since that would allow access to all its children). */ ref fileAccessor = - AllowListInputAccessor::create( - makeFSInputAccessor(path), - std::set { wd.files }, - std::move(makeNotAllowedError)); - if (exportIgnore) { + wd.files.empty() + ? makeEmptyInputAccessor() + : AllowListInputAccessor::create( + makeFSInputAccessor(path), + std::set { wd.files }, + std::move(makeNotAllowedError)).cast(); + if (exportIgnore) return make_ref(self, fileAccessor, std::nullopt); - } - else { + else return fileAccessor; - } +} + +ref GitRepoImpl::getFileSystemObjectSink() +{ + return make_ref(ref(shared_from_this())); } std::vector> GitRepoImpl::getSubmodules(const Hash & rev, bool exportIgnore) @@ -781,7 +999,7 @@ std::vector> GitRepoImpl::getSubmodules auto rawAccessor = getRawAccessor(rev); - for (auto & submodule : parseSubmodules(CanonPath(pathTemp))) { + for (auto & submodule : parseSubmodules(pathTemp)) { auto rev = rawAccessor->getSubmoduleRev(submodule.path); result.push_back({std::move(submodule), rev}); } @@ -789,5 +1007,11 @@ std::vector> GitRepoImpl::getSubmodules return result; } +ref getTarballCache() +{ + static auto repoDir = std::filesystem::path(getCacheDir()) / "nix" / "tarball-cache"; + + return GitRepo::openRepo(repoDir, true, true); +} } diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh index 768554780..fbb2d947b 100644 --- a/src/libfetchers/git-utils.hh +++ b/src/libfetchers/git-utils.hh @@ -2,17 +2,26 @@ #include "filtering-input-accessor.hh" #include "input-accessor.hh" +#include "fs-sink.hh" namespace nix { namespace fetchers { struct PublicKey; } +struct GitFileSystemObjectSink : FileSystemObjectSink +{ + /** + * Flush builder and return a final Git hash. + */ + virtual Hash sync() = 0; +}; + struct GitRepo { virtual ~GitRepo() { } - static ref openRepo(const CanonPath & path, bool create = false, bool bare = false); + static ref openRepo(const std::filesystem::path & path, bool create = false, bool bare = false); virtual uint64_t getRevCount(const Hash & rev) = 0; @@ -64,18 +73,14 @@ struct GitRepo const std::string & url, const std::string & base) = 0; - struct TarballInfo - { - Hash treeHash; - time_t lastModified; - }; - virtual bool hasObject(const Hash & oid) = 0; virtual ref getAccessor(const Hash & rev, bool exportIgnore) = 0; virtual ref getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllowedError makeNotAllowedError) = 0; + virtual ref getFileSystemObjectSink() = 0; + virtual void fetch( const std::string & url, const std::string & refspec, @@ -88,6 +93,14 @@ struct GitRepo virtual void verifyCommit( const Hash & rev, const std::vector & publicKeys) = 0; + + /** + * Given a Git tree hash, compute the hash of its NAR + * serialisation. This is memoised on-disk. + */ + virtual Hash treeHashToNarHash(const Hash & treeHash) = 0; }; +ref getTarballCache(); + } diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index f9a1cb1bc..87d114276 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -158,6 +158,8 @@ std::vector getPublicKeys(const Attrs & attrs) } // end namespace +static const Hash nullRev{HashAlgorithm::SHA1}; + struct GitInputScheme : InputScheme { std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override @@ -319,7 +321,7 @@ struct GitInputScheme : InputScheme if (!repoInfo.isLocal) throw Error("cannot commit '%s' to Git repository '%s' because it's not a working tree", path, input.to_string()); - writeFile((CanonPath(repoInfo.url) + path).abs(), contents); + writeFile((CanonPath(repoInfo.url) / path).abs(), contents); auto result = runProgram(RunOptions { .program = "git", @@ -415,7 +417,7 @@ struct GitInputScheme : InputScheme // If this is a local directory and no ref or revision is // given, then allow the use of an unclean working tree. if (!input.getRef() && !input.getRev() && repoInfo.isLocal) - repoInfo.workdirInfo = GitRepo::openRepo(CanonPath(repoInfo.url))->getWorkdirInfo(); + repoInfo.workdirInfo = GitRepo::openRepo(repoInfo.url)->getWorkdirInfo(); return repoInfo; } @@ -429,7 +431,7 @@ struct GitInputScheme : InputScheme if (auto res = cache->lookup(key)) return getIntAttr(*res, "lastModified"); - auto lastModified = GitRepo::openRepo(CanonPath(repoDir))->getLastModified(rev); + auto lastModified = GitRepo::openRepo(repoDir)->getLastModified(rev); cache->upsert(key, Attrs{{"lastModified", lastModified}}); @@ -447,7 +449,7 @@ struct GitInputScheme : InputScheme Activity act(*logger, lvlChatty, actUnknown, fmt("getting Git revision count of '%s'", repoInfo.url)); - auto revCount = GitRepo::openRepo(CanonPath(repoDir))->getRevCount(rev); + auto revCount = GitRepo::openRepo(repoDir)->getRevCount(rev); cache->upsert(key, Attrs{{"revCount", revCount}}); @@ -457,7 +459,7 @@ struct GitInputScheme : InputScheme std::string getDefaultRef(const RepoInfo & repoInfo) const { auto head = repoInfo.isLocal - ? GitRepo::openRepo(CanonPath(repoInfo.url))->getWorkdirRef() + ? GitRepo::openRepo(repoInfo.url)->getWorkdirRef() : readHeadCached(repoInfo.url); if (!head) { warn("could not read HEAD ref from repo at '%s', using 'master'", repoInfo.url); @@ -510,7 +512,7 @@ struct GitInputScheme : InputScheme if (repoInfo.isLocal) { repoDir = repoInfo.url; if (!input.getRev()) - input.attrs.insert_or_assign("rev", GitRepo::openRepo(CanonPath(repoDir))->resolveRef(ref).gitRev()); + input.attrs.insert_or_assign("rev", GitRepo::openRepo(repoDir)->resolveRef(ref).gitRev()); } else { Path cacheDir = getCachePath(repoInfo.url, getShallowAttr(input)); repoDir = cacheDir; @@ -519,7 +521,7 @@ struct GitInputScheme : InputScheme createDirs(dirOf(cacheDir)); PathLocks cacheDirLock({cacheDir}); - auto repo = GitRepo::openRepo(CanonPath(cacheDir), true, true); + auto repo = GitRepo::openRepo(cacheDir, true, true); Path localRefFile = ref.compare(0, 5, "refs/") == 0 @@ -588,7 +590,7 @@ struct GitInputScheme : InputScheme // cache dir lock is removed at scope end; we will only use read-only operations on specific revisions in the remainder } - auto repo = GitRepo::openRepo(CanonPath(repoDir)); + auto repo = GitRepo::openRepo(repoDir); auto isShallow = repo->isShallow(); @@ -664,7 +666,7 @@ struct GitInputScheme : InputScheme for (auto & submodule : repoInfo.workdirInfo.submodules) repoInfo.workdirInfo.files.insert(submodule.path); - auto repo = GitRepo::openRepo(CanonPath(repoInfo.url), false, false); + auto repo = GitRepo::openRepo(repoInfo.url, false, false); auto exportIgnore = getExportIgnoreAttr(input); @@ -680,7 +682,7 @@ struct GitInputScheme : InputScheme std::map> mounts; for (auto & submodule : repoInfo.workdirInfo.submodules) { - auto submodulePath = CanonPath(repoInfo.url) + submodule.path; + auto submodulePath = CanonPath(repoInfo.url) / submodule.path; fetchers::Attrs attrs; attrs.insert_or_assign("type", "git"); attrs.insert_or_assign("url", submodulePath.abs()); @@ -703,15 +705,17 @@ struct GitInputScheme : InputScheme } if (!repoInfo.workdirInfo.isDirty) { - auto repo = GitRepo::openRepo(CanonPath(repoInfo.url)); + auto repo = GitRepo::openRepo(repoInfo.url); if (auto ref = repo->getWorkdirRef()) input.attrs.insert_or_assign("ref", *ref); - auto rev = repoInfo.workdirInfo.headRev.value(); + /* Return a rev of 000... if there are no commits yet. */ + auto rev = repoInfo.workdirInfo.headRev.value_or(nullRev); input.attrs.insert_or_assign("rev", rev.gitRev()); - input.attrs.insert_or_assign("revCount", getRevCount(repoInfo, repoInfo.url, rev)); + input.attrs.insert_or_assign("revCount", + rev == nullRev ? 0 : getRevCount(repoInfo, repoInfo.url, rev)); verifyCommit(input, repo); } else { @@ -733,8 +737,6 @@ struct GitInputScheme : InputScheme ? getLastModified(repoInfo, repoInfo.url, *repoInfo.workdirInfo.headRev) : 0); - input.locked = true; // FIXME - return {accessor, std::move(input)}; } @@ -771,6 +773,11 @@ struct GitInputScheme : InputScheme else return std::nullopt; } + + bool isLocked(const Input & input) const override + { + return (bool) input.getRev(); + } }; static auto rGitInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 498e41357..a48c99a0b 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -8,6 +8,8 @@ #include "fetchers.hh" #include "fetch-settings.hh" #include "tarball.hh" +#include "tarfile.hh" +#include "git-utils.hh" #include #include @@ -180,49 +182,111 @@ struct GitArchiveInputScheme : InputScheme return headers; } - virtual Hash getRevFromRef(nix::ref store, const Input & input) const = 0; + struct RefInfo + { + Hash rev; + std::optional treeHash; + }; + + virtual RefInfo getRevFromRef(nix::ref store, const Input & input) const = 0; virtual DownloadUrl getDownloadUrl(const Input & input) const = 0; - std::pair fetch(ref store, const Input & _input) override + struct TarballInfo { - Input input(_input); + Hash treeHash; + time_t lastModified; + }; + std::pair downloadArchive(ref store, Input input) const + { if (!maybeGetStrAttr(input.attrs, "ref")) input.attrs.insert_or_assign("ref", "HEAD"); + std::optional upstreamTreeHash; + auto rev = input.getRev(); - if (!rev) rev = getRevFromRef(store, input); + if (!rev) { + auto refInfo = getRevFromRef(store, input); + rev = refInfo.rev; + upstreamTreeHash = refInfo.treeHash; + debug("HEAD revision for '%s' is %s", input.to_string(), refInfo.rev.gitRev()); + } input.attrs.erase("ref"); input.attrs.insert_or_assign("rev", rev->gitRev()); - Attrs lockedAttrs({ - {"type", "git-tarball"}, - {"rev", rev->gitRev()}, - }); + auto cache = getCache(); - if (auto res = getCache()->lookup(*store, lockedAttrs)) { - input.attrs.insert_or_assign("lastModified", getIntAttr(res->first, "lastModified")); - return {std::move(res->second), input}; + Attrs treeHashKey{{"_what", "gitRevToTreeHash"}, {"rev", rev->gitRev()}}; + Attrs lastModifiedKey{{"_what", "gitRevToLastModified"}, {"rev", rev->gitRev()}}; + + if (auto treeHashAttrs = cache->lookup(treeHashKey)) { + if (auto lastModifiedAttrs = cache->lookup(lastModifiedKey)) { + auto treeHash = getRevAttr(*treeHashAttrs, "treeHash"); + auto lastModified = getIntAttr(*lastModifiedAttrs, "lastModified"); + if (getTarballCache()->hasObject(treeHash)) + return {std::move(input), TarballInfo { .treeHash = treeHash, .lastModified = (time_t) lastModified }}; + else + debug("Git tree with hash '%s' has disappeared from the cache, refetching...", treeHash.gitRev()); + } } + /* Stream the tarball into the tarball cache. */ auto url = getDownloadUrl(input); - auto result = downloadTarball(store, url.url, input.getName(), true, url.headers); + auto source = sinkToSource([&](Sink & sink) { + FileTransferRequest req(url.url); + req.headers = url.headers; + getFileTransfer()->download(std::move(req), sink); + }); - input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified)); + TarArchive archive { *source }; + auto parseSink = getTarballCache()->getFileSystemObjectSink(); + auto lastModified = unpackTarfileToSink(archive, *parseSink); - getCache()->add( - *store, - lockedAttrs, - { - {"rev", rev->gitRev()}, - {"lastModified", uint64_t(result.lastModified)} - }, - result.storePath, - true); + TarballInfo tarballInfo { + .treeHash = parseSink->sync(), + .lastModified = lastModified + }; - return {result.storePath, input}; + cache->upsert(treeHashKey, Attrs{{"treeHash", tarballInfo.treeHash.gitRev()}}); + cache->upsert(lastModifiedKey, Attrs{{"lastModified", (uint64_t) tarballInfo.lastModified}}); + + #if 0 + if (upstreamTreeHash != tarballInfo.treeHash) + warn( + "Git tree hash mismatch for revision '%s' of '%s': " + "expected '%s', got '%s'. " + "This can happen if the Git repository uses submodules.", + rev->gitRev(), input.to_string(), upstreamTreeHash->gitRev(), tarballInfo.treeHash.gitRev()); + #endif + + return {std::move(input), tarballInfo}; + } + + std::pair, Input> getAccessor(ref store, const Input & _input) const override + { + auto [input, tarballInfo] = downloadArchive(store, _input); + + input.attrs.insert_or_assign("treeHash", tarballInfo.treeHash.gitRev()); + input.attrs.insert_or_assign("lastModified", uint64_t(tarballInfo.lastModified)); + + auto accessor = getTarballCache()->getAccessor(tarballInfo.treeHash, false); + + accessor->setPathDisplay("«" + input.to_string() + "»"); + + accessor->fingerprint = input.getFingerprint(store); + + return {accessor, input}; + } + + bool isLocked(const Input & input) const override + { + /* Since we can't verify the integrity of the tarball from the + Git revision alone, we also require a NAR hash for + locking. FIXME: in the future, we may want to require a Git + tree hash instead of a NAR hash. */ + return input.getRev().has_value() && input.getNarHash().has_value(); } std::optional experimentalFeature() const override @@ -269,7 +333,7 @@ struct GitHubInputScheme : GitArchiveInputScheme return getStrAttr(input.attrs, "repo"); } - Hash getRevFromRef(nix::ref store, const Input & input) const override + RefInfo getRevFromRef(nix::ref store, const Input & input) const override { auto host = getHost(input); auto url = fmt( @@ -284,9 +348,11 @@ struct GitHubInputScheme : GitArchiveInputScheme readFile( store->toRealPath( downloadFile(store, url, "source", false, headers).storePath))); - auto rev = Hash::parseAny(std::string { json["sha"] }, HashAlgorithm::SHA1); - debug("HEAD revision for '%s' is %s", url, rev.gitRev()); - return rev; + + return RefInfo { + .rev = Hash::parseAny(std::string { json["sha"] }, HashAlgorithm::SHA1), + .treeHash = Hash::parseAny(std::string { json["commit"]["tree"]["sha"] }, HashAlgorithm::SHA1) + }; } DownloadUrl getDownloadUrl(const Input & input) const override @@ -343,7 +409,7 @@ struct GitLabInputScheme : GitArchiveInputScheme return std::make_pair(token.substr(0,fldsplit), token.substr(fldsplit+1)); } - Hash getRevFromRef(nix::ref store, const Input & input) const override + RefInfo getRevFromRef(nix::ref store, const Input & input) const override { auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); // See rate limiting note below @@ -356,9 +422,10 @@ struct GitLabInputScheme : GitArchiveInputScheme readFile( store->toRealPath( downloadFile(store, url, "source", false, headers).storePath))); - auto rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1); - debug("HEAD revision for '%s' is %s", url, rev.gitRev()); - return rev; + + return RefInfo { + .rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1) + }; } DownloadUrl getDownloadUrl(const Input & input) const override @@ -402,7 +469,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme // Once it is implemented, however, should work as expected. } - Hash getRevFromRef(nix::ref store, const Input & input) const override + RefInfo getRevFromRef(nix::ref store, const Input & input) const override { // TODO: In the future, when the sourcehut graphql API is implemented for mercurial // and with anonymous access, this method should use it instead. @@ -445,12 +512,12 @@ struct SourceHutInputScheme : GitArchiveInputScheme id = parsedLine->target; } - if(!id) + if (!id) throw BadURL("in '%d', couldn't find ref '%d'", input.to_string(), ref); - auto rev = Hash::parseAny(*id, HashAlgorithm::SHA1); - debug("HEAD revision for '%s' is %s", fmt("%s/%s", base_url, ref), rev.gitRev()); - return rev; + return RefInfo { + .rev = Hash::parseAny(*id, HashAlgorithm::SHA1) + }; } DownloadUrl getDownloadUrl(const Input & input) const override diff --git a/src/libfetchers/memory-input-accessor.cc b/src/libfetchers/memory-input-accessor.cc index 88a2e34e8..34a801f67 100644 --- a/src/libfetchers/memory-input-accessor.cc +++ b/src/libfetchers/memory-input-accessor.cc @@ -20,4 +20,10 @@ ref makeMemoryInputAccessor() return make_ref(); } +ref makeEmptyInputAccessor() +{ + static auto empty = makeMemoryInputAccessor().cast(); + return empty; +} + } diff --git a/src/libfetchers/memory-input-accessor.hh b/src/libfetchers/memory-input-accessor.hh index 508b07722..63afadd2a 100644 --- a/src/libfetchers/memory-input-accessor.hh +++ b/src/libfetchers/memory-input-accessor.hh @@ -13,4 +13,6 @@ struct MemoryInputAccessor : InputAccessor ref makeMemoryInputAccessor(); +ref makeEmptyInputAccessor(); + } diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 9982389ab..a5f55a44e 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -141,7 +141,7 @@ struct MercurialInputScheme : InputScheme if (!isLocal) throw Error("cannot commit '%s' to Mercurial repository '%s' because it's not a working tree", path, input.to_string()); - auto absPath = CanonPath(repoPath) + path; + auto absPath = CanonPath(repoPath) / path; writeFile(absPath.abs(), contents); @@ -347,6 +347,11 @@ struct MercurialInputScheme : InputScheme return makeResult(infoAttrs, std::move(storePath)); } + bool isLocked(const Input & input) const override + { + return (bool) input.getRev(); + } + std::optional getFingerprint(ref store, const Input & input) const override { if (auto rev = input.getRev()) diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index f9b973320..276fd1b36 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -84,7 +84,12 @@ struct PathInputScheme : InputScheme std::string_view contents, std::optional commitMsg) const override { - writeFile((CanonPath(getAbsPath(input)) + path).abs(), contents); + writeFile((CanonPath(getAbsPath(input)) / path).abs(), contents); + } + + bool isLocked(const Input & input) const override + { + return (bool) input.getNarHash(); } CanonPath getAbsPath(const Input & input) const diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 3b7709440..f08509cb7 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -9,6 +9,9 @@ #include "types.hh" #include "split.hh" #include "posix-source-accessor.hh" +#include "fs-input-accessor.hh" +#include "store-api.hh" +#include "git-utils.hh" namespace nix::fetchers { @@ -57,10 +60,8 @@ DownloadFileResult downloadFile( throw; } - // FIXME: write to temporary file. Attrs infoAttrs({ {"etag", res.etag}, - {"url", res.effectiveUri}, }); if (res.immutableUrl) @@ -91,96 +92,102 @@ DownloadFileResult downloadFile( storePath = std::move(info.path); } - getCache()->add( - *store, - inAttrs, - infoAttrs, - *storePath, - locked); - - if (url != res.effectiveUri) + /* Cache metadata for all URLs in the redirect chain. */ + for (auto & url : res.urls) { + inAttrs.insert_or_assign("url", url); + infoAttrs.insert_or_assign("url", *res.urls.rbegin()); getCache()->add( *store, - { - {"type", "file"}, - {"url", res.effectiveUri}, - {"name", name}, - }, + inAttrs, infoAttrs, *storePath, locked); + } return { .storePath = std::move(*storePath), .etag = res.etag, - .effectiveUrl = res.effectiveUri, + .effectiveUrl = *res.urls.rbegin(), .immutableUrl = res.immutableUrl, }; } DownloadTarballResult downloadTarball( - ref store, const std::string & url, - const std::string & name, - bool locked, const Headers & headers) { Attrs inAttrs({ - {"type", "tarball"}, + {"_what", "tarballCache"}, {"url", url}, - {"name", name}, }); - auto cached = getCache()->lookupExpired(*store, inAttrs); + auto cached = getCache()->lookupExpired(inAttrs); + + auto attrsToResult = [&](const Attrs & infoAttrs) + { + auto treeHash = getRevAttr(infoAttrs, "treeHash"); + return DownloadTarballResult { + .treeHash = treeHash, + .lastModified = (time_t) getIntAttr(infoAttrs, "lastModified"), + .immutableUrl = maybeGetStrAttr(infoAttrs, "immutableUrl"), + .accessor = getTarballCache()->getAccessor(treeHash, false), + }; + }; + + if (cached && !getTarballCache()->hasObject(getRevAttr(cached->infoAttrs, "treeHash"))) + cached.reset(); if (cached && !cached->expired) - return { - .storePath = std::move(cached->storePath), - .lastModified = (time_t) getIntAttr(cached->infoAttrs, "lastModified"), - .immutableUrl = maybeGetStrAttr(cached->infoAttrs, "immutableUrl"), - }; + /* We previously downloaded this tarball and it's younger than + `tarballTtl`, so no need to check the server. */ + return attrsToResult(cached->infoAttrs); - auto res = downloadFile(store, url, name, locked, headers); + auto _res = std::make_shared>(); - std::optional unpackedStorePath; - time_t lastModified; - - if (cached && res.etag != "" && getStrAttr(cached->infoAttrs, "etag") == res.etag) { - unpackedStorePath = std::move(cached->storePath); - lastModified = getIntAttr(cached->infoAttrs, "lastModified"); - } else { - Path tmpDir = createTempDir(); - AutoDelete autoDelete(tmpDir, true); - unpackTarfile(store->toRealPath(res.storePath), tmpDir); - auto members = readDirectory(tmpDir); - if (members.size() != 1) - throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); - auto topDir = tmpDir + "/" + members.begin()->name; - lastModified = lstat(topDir).st_mtime; - PosixSourceAccessor accessor; - unpackedStorePath = store->addToStore(name, accessor, CanonPath { topDir }, FileIngestionMethod::Recursive, HashAlgorithm::SHA256, {}, defaultPathFilter, NoRepair); - } - - Attrs infoAttrs({ - {"lastModified", uint64_t(lastModified)}, - {"etag", res.etag}, + auto source = sinkToSource([&](Sink & sink) { + FileTransferRequest req(url); + req.expectedETag = cached ? getStrAttr(cached->infoAttrs, "etag") : ""; + getFileTransfer()->download(std::move(req), sink, + [_res](FileTransferResult r) + { + *_res->lock() = r; + }); }); - if (res.immutableUrl) - infoAttrs.emplace("immutableUrl", *res.immutableUrl); + // TODO: fall back to cached value if download fails. - getCache()->add( - *store, - inAttrs, - infoAttrs, - *unpackedStorePath, - locked); + /* Note: if the download is cached, `importTarball()` will receive + no data, which causes it to import an empty tarball. */ + TarArchive archive { *source }; + auto parseSink = getTarballCache()->getFileSystemObjectSink(); + auto lastModified = unpackTarfileToSink(archive, *parseSink); - return { - .storePath = std::move(*unpackedStorePath), - .lastModified = lastModified, - .immutableUrl = res.immutableUrl, - }; + auto res(_res->lock()); + + Attrs infoAttrs; + + if (res->cached) { + /* The server says that the previously downloaded version is + still current. */ + infoAttrs = cached->infoAttrs; + } else { + infoAttrs.insert_or_assign("etag", res->etag); + infoAttrs.insert_or_assign("treeHash", parseSink->sync().gitRev()); + infoAttrs.insert_or_assign("lastModified", uint64_t(lastModified)); + if (res->immutableUrl) + infoAttrs.insert_or_assign("immutableUrl", *res->immutableUrl); + } + + /* Insert a cache entry for every URL in the redirect chain. */ + for (auto & url : res->urls) { + inAttrs.insert_or_assign("url", url); + getCache()->upsert(inAttrs, infoAttrs); + } + + // FIXME: add a cache entry for immutableUrl? That could allow + // cache poisoning. + + return attrsToResult(infoAttrs); } // An input scheme corresponding to a curl-downloadable resource. @@ -198,6 +205,8 @@ struct CurlInputScheme : InputScheme virtual bool isValidURL(const ParsedURL & url, bool requireTree) const = 0; + static const std::set specialParams; + std::optional inputFromURL(const ParsedURL & _url, bool requireTree) const override { if (!isValidURL(_url, requireTree)) @@ -220,8 +229,17 @@ struct CurlInputScheme : InputScheme if (auto n = string2Int(*i)) input.attrs.insert_or_assign("revCount", *n); - url.query.erase("rev"); - url.query.erase("revCount"); + if (auto i = get(url.query, "lastModified")) + if (auto n = string2Int(*i)) + input.attrs.insert_or_assign("lastModified", *n); + + /* The URL query parameters serve two roles: specifying fetch + settings for Nix itself, and arbitrary data as part of the + HTTP request. Now that we've processed the Nix-specific + attributes above, remove them so we don't also send them as + part of the HTTP request. */ + for (auto & param : allowedAttrs()) + url.query.erase(param); input.attrs.insert_or_assign("type", std::string { schemeName() }); input.attrs.insert_or_assign("url", url.to_string()); @@ -260,6 +278,11 @@ struct CurlInputScheme : InputScheme url.query.insert_or_assign("narHash", narHash->to_string(HashFormat::SRI, true)); return url; } + + bool isLocked(const Input & input) const override + { + return (bool) input.getNarHash(); + } }; struct FileInputScheme : CurlInputScheme @@ -275,10 +298,24 @@ struct FileInputScheme : CurlInputScheme : (!requireTree && !hasTarballExtension(url.path))); } - std::pair fetch(ref store, const Input & input) override + std::pair, Input> getAccessor(ref store, const Input & _input) const override { + auto input(_input); + + /* Unlike TarballInputScheme, this stores downloaded files in + the Nix store directly, since there is little deduplication + benefit in using the Git cache for single big files like + tarballs. */ auto file = downloadFile(store, getStrAttr(input.attrs, "url"), input.getName(), false); - return {std::move(file.storePath), input}; + + auto narHash = store->queryPathInfo(file.storePath)->narHash; + input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); + + auto accessor = makeStorePathAccessor(store, file.storePath); + + accessor->setPathDisplay("«" + input.to_string() + "»"); + + return {accessor, input}; } }; @@ -296,11 +333,13 @@ struct TarballInputScheme : CurlInputScheme : (requireTree || hasTarballExtension(url.path))); } - std::pair fetch(ref store, const Input & _input) override + std::pair, Input> getAccessor(ref store, const Input & _input) const override { - Input input(_input); - auto url = getStrAttr(input.attrs, "url"); - auto result = downloadTarball(store, url, input.getName(), false); + auto input(_input); + + auto result = downloadTarball(getStrAttr(input.attrs, "url"), {}); + + result.accessor->setPathDisplay("«" + input.to_string() + "»"); if (result.immutableUrl) { auto immutableInput = Input::fromURL(*result.immutableUrl); @@ -314,7 +353,10 @@ struct TarballInputScheme : CurlInputScheme if (result.lastModified && !input.attrs.contains("lastModified")) input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified)); - return {result.storePath, std::move(input)}; + input.attrs.insert_or_assign("narHash", + getTarballCache()->treeHashToNarHash(result.treeHash).to_string(HashFormat::SRI, true)); + + return {result.accessor, input}; } }; diff --git a/src/libfetchers/tarball.hh b/src/libfetchers/tarball.hh index 9e6b50b31..77ad3bf09 100644 --- a/src/libfetchers/tarball.hh +++ b/src/libfetchers/tarball.hh @@ -2,11 +2,13 @@ #include "types.hh" #include "path.hh" +#include "hash.hh" #include namespace nix { class Store; +struct InputAccessor; } namespace nix::fetchers { @@ -28,16 +30,18 @@ DownloadFileResult downloadFile( struct DownloadTarballResult { - StorePath storePath; + Hash treeHash; time_t lastModified; std::optional immutableUrl; + ref accessor; }; +/** + * Download and import a tarball into the Git cache. The result is the + * Git tree hash of the root directory. + */ DownloadTarballResult downloadTarball( - ref store, const std::string & url, - const std::string & name, - bool locked, const Headers & headers = {}); } diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 862ef355b..7bced0aa4 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -340,7 +340,7 @@ int handleExceptions(const std::string & programName, std::function fun) return 1; } catch (BaseError & e) { logError(e.info()); - return e.status; + return e.info().status; } catch (std::bad_alloc & e) { printError(error + "out of memory"); return 1; @@ -408,6 +408,4 @@ PrintFreed::~PrintFreed() showBytes(results.bytesFreed)); } -Exit::~Exit() { } - } diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index c68f6cd83..99c3dffab 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -7,6 +7,7 @@ #include "common-args.hh" #include "path.hh" #include "derived-path.hh" +#include "exit.hh" #include @@ -15,15 +16,6 @@ namespace nix { -class Exit : public std::exception -{ -public: - int status; - Exit() : status(0) { } - Exit(int status) : status(status) { } - virtual ~Exit(); -}; - int handleExceptions(const std::string & programName, std::function fun); /** diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index ea1279e2e..bea2bb370 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -235,14 +235,14 @@ ref BinaryCacheStore::addToStoreCommon( std::regex regex2("^[0-9a-f]{38}\\.debug$"); for (auto & [s1, _type] : narAccessor->readDirectory(buildIdDir)) { - auto dir = buildIdDir + s1; + auto dir = buildIdDir / s1; if (narAccessor->lstat(dir).type != SourceAccessor::tDirectory || !std::regex_match(s1, regex1)) continue; for (auto & [s2, _type] : narAccessor->readDirectory(dir)) { - auto debugPath = dir + s2; + auto debugPath = dir / s2; if (narAccessor->lstat(debugPath).type != SourceAccessor::tRegular || !std::regex_match(s2, regex2)) @@ -305,7 +305,8 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource StorePath BinaryCacheStore::addToStoreFromDump( Source & dump, std::string_view name, - ContentAddressMethod method, + FileSerialisationMethod dumpMethod, + ContentAddressMethod hashMethod, HashAlgorithm hashAlgo, const StorePathSet & references, RepairFlag repair) @@ -313,17 +314,27 @@ StorePath BinaryCacheStore::addToStoreFromDump( std::optional caHash; std::string nar; + // Calculating Git hash from NAR stream not yet implemented. May not + // be possible to implement in single-pass if the NAR is in an + // inconvenient order. Could fetch after uploading, however. + if (hashMethod.getFileIngestionMethod() == FileIngestionMethod::Git) + unsupported("addToStoreFromDump"); + if (auto * dump2p = dynamic_cast(&dump)) { auto & dump2 = *dump2p; // Hack, this gives us a "replayable" source so we can compute // multiple hashes more easily. - caHash = hashString(HashAlgorithm::SHA256, dump2.s); - switch (method.getFileIngestionMethod()) { - case FileIngestionMethod::Recursive: + // + // Only calculate if the dump is in the right format, however. + if (static_cast(dumpMethod) == hashMethod.getFileIngestionMethod()) + caHash = hashString(HashAlgorithm::SHA256, dump2.s); + switch (dumpMethod) { + case FileSerialisationMethod::Recursive: // The dump is already NAR in this case, just use it. nar = dump2.s; break; - case FileIngestionMethod::Flat: + case FileSerialisationMethod::Flat: + { // The dump is Flat, so we need to convert it to NAR with a // single file. StringSink s; @@ -331,10 +342,11 @@ StorePath BinaryCacheStore::addToStoreFromDump( nar = std::move(s.s); break; } + } } else { // Otherwise, we have to do th same hashing as NAR so our single // hash will suffice for both purposes. - if (method != FileIngestionMethod::Recursive || hashAlgo != HashAlgorithm::SHA256) + if (dumpMethod != FileSerialisationMethod::Recursive || hashAlgo != HashAlgorithm::SHA256) unsupported("addToStoreFromDump"); } StringSource narDump { nar }; @@ -349,7 +361,7 @@ StorePath BinaryCacheStore::addToStoreFromDump( *this, name, ContentAddressWithReferences::fromParts( - method, + hashMethod, caHash ? *caHash : nar.first, { .others = references, @@ -450,7 +462,7 @@ StorePath BinaryCacheStore::addToStore( non-recursive+sha256 so we can just use the default implementation of this method in terms of addToStoreFromDump. */ - auto h = hashPath(accessor, path, method.getFileIngestionMethod(), hashAlgo, filter).first; + auto h = hashPath(accessor, path, method.getFileIngestionMethod(), hashAlgo, filter); auto source = sinkToSource([&](Sink & sink) { accessor.dumpPath(path, sink, filter); diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 00ab73905..7c2828309 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -125,7 +125,8 @@ public: StorePath addToStoreFromDump( Source & dump, std::string_view name, - ContentAddressMethod method, + FileSerialisationMethod dumpMethod, + ContentAddressMethod hashMethod, HashAlgorithm hashAlgo, const StorePathSet & references, RepairFlag repair) override; @@ -147,7 +148,7 @@ public: void narFromPath(const StorePath & path, Sink & sink) override; - ref getFSAccessor(bool requireValidPath) override; + ref getFSAccessor(bool requireValidPath = true) override; void addSignatures(const StorePath & storePath, const StringSet & sigs) override; diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 00cbf4228..29bf2852f 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -708,7 +708,7 @@ void DerivationGoal::tryToBuild() if (!outputLocks.lockPaths(lockFiles, "", false)) { if (!actLock) actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for lock on %s", yellowtxt(showPaths(lockFiles)))); + fmt("waiting for lock on %s", Magenta(showPaths(lockFiles)))); worker.waitForAWhile(shared_from_this()); return; } @@ -762,7 +762,7 @@ void DerivationGoal::tryToBuild() the wake-up timeout expires. */ if (!actLock) actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for a machine to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); + fmt("waiting for a machine to build '%s'", Magenta(worker.store.printStorePath(drvPath)))); worker.waitForAWhile(shared_from_this()); outputLocks.unlock(); return; @@ -780,9 +780,13 @@ void DerivationGoal::tryToBuild() void DerivationGoal::tryLocalBuild() { throw Error( - "unable to build with a primary store that isn't a local store; " - "either pass a different '--store' or enable remote builds." - "\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html"); + R"( + Unable to build with a primary store that isn't a local store; + either pass a different '--store' or enable remote builds. + + For more information check 'man nix.conf' and search for '/machines'. + )" + ); } @@ -891,7 +895,7 @@ void runPostBuildHook( if (hook == "") return; - Activity act(logger, lvlInfo, actPostBuildHook, + Activity act(logger, lvlTalkative, actPostBuildHook, fmt("running post-build-hook '%s'", settings.postBuildHook), Logger::Fields{store.printStorePath(drvPath)}); PushActivity pact(act.id); @@ -987,7 +991,7 @@ void DerivationGoal::buildDone() diskFull |= cleanupDecideWhetherDiskFull(); auto msg = fmt("builder for '%s' %s", - yellowtxt(worker.store.printStorePath(drvPath)), + Magenta(worker.store.printStorePath(drvPath)), statusToString(status)); if (!logger->isVerbose() && !logTail.empty()) { @@ -1523,7 +1527,7 @@ void DerivationGoal::done( outputLocks.unlock(); buildResult.status = status; if (ex) - buildResult.errorMsg = fmt("%s", normaltxt(ex->info().msg)); + buildResult.errorMsg = fmt("%s", Uncolored(ex->info().msg)); if (buildResult.status == BuildResult::TimedOut) worker.timedOut = true; if (buildResult.status == BuildResult::PermanentFailure) diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 7f0a05d5d..d4bead28e 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -33,7 +33,7 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod } if (failed.size() == 1 && ex) { - ex->status = worker.failingExitStatus(); + ex->withExitStatus(worker.failingExitStatus()); throw std::move(*ex); } else if (!failed.empty()) { if (ex) logError(ex->info()); @@ -104,7 +104,7 @@ void Store::ensurePath(const StorePath & path) if (goal->exitCode != Goal::ecSuccess) { if (goal->ex) { - goal->ex->status = worker.failingExitStatus(); + goal->ex->withExitStatus(worker.failingExitStatus()); throw std::move(*goal->ex); } else throw Error(worker.failingExitStatus(), "path '%s' does not exist and cannot be created", printStorePath(path)); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 2ba8be7d6..a9b8de123 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -8,6 +8,7 @@ #include "finally.hh" #include "util.hh" #include "archive.hh" +#include "git.hh" #include "compression.hh" #include "daemon.hh" #include "topo-sort.hh" @@ -92,7 +93,7 @@ void handleDiffHook( } catch (Error & error) { ErrorInfo ei = error.info(); // FIXME: wrap errors. - ei.msg = hintfmt("diff hook execution failed: %s", ei.msg.str()); + ei.msg = HintFmt("diff hook execution failed: %s", ei.msg.str()); logError(ei); } } @@ -232,7 +233,7 @@ void LocalDerivationGoal::tryLocalBuild() if (!buildUser) { if (!actLock) actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for a free build user ID for '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); + fmt("waiting for a free build user ID for '%s'", Magenta(worker.store.printStorePath(drvPath)))); worker.waitForAWhile(shared_from_this()); return; } @@ -1311,12 +1312,13 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In StorePath addToStoreFromDump( Source & dump, std::string_view name, - ContentAddressMethod method, + FileSerialisationMethod dumpMethod, + ContentAddressMethod hashMethod, HashAlgorithm hashAlgo, const StorePathSet & references, RepairFlag repair) override { - auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, references, repair); + auto path = next->addToStoreFromDump(dump, name, dumpMethod, hashMethod, hashAlgo, references, repair); goal.addDependency(path); return path; } @@ -2130,16 +2132,17 @@ void LocalDerivationGoal::runChild() try { logger = makeJSONLogger(*logger); - BasicDerivation & drv2(*drv); - for (auto & e : drv2.env) - e.second = rewriteStrings(e.second, inputRewrites); + std::map outputs; + for (auto & e : drv->outputs) + outputs.insert_or_assign(e.first, + worker.store.printStorePath(scratchOutputs.at(e.first))); if (drv->builder == "builtin:fetchurl") - builtinFetchurl(drv2, netrcData); + builtinFetchurl(*drv, outputs, netrcData); else if (drv->builder == "builtin:buildenv") - builtinBuildenv(drv2); + builtinBuildenv(*drv, outputs); else if (drv->builder == "builtin:unpack-channel") - builtinUnpackChannel(drv2); + builtinUnpackChannel(*drv, outputs); else throw Error("unsupported builtin builder '%1%'", drv->builder.substr(8)); _exit(0); @@ -2456,15 +2459,28 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() rewriteOutput(outputRewrites); /* FIXME optimize and deduplicate with addToStore */ std::string oldHashPart { scratchPath->hashPart() }; - auto got = ({ - HashModuloSink caSink { outputHash.hashAlgo, oldHashPart }; + auto got = [&]{ PosixSourceAccessor accessor; - dumpPath( - accessor, CanonPath { actualPath }, - caSink, - outputHash.method.getFileIngestionMethod()); - caSink.finish().first; - }); + auto fim = outputHash.method.getFileIngestionMethod(); + switch (fim) { + case FileIngestionMethod::Flat: + case FileIngestionMethod::Recursive: + { + HashModuloSink caSink { outputHash.hashAlgo, oldHashPart }; + auto fim = outputHash.method.getFileIngestionMethod(); + dumpPath( + accessor, CanonPath { actualPath }, + caSink, + (FileSerialisationMethod) fim); + return caSink.finish().first; + } + case FileIngestionMethod::Git: { + return git::dumpHash( + outputHash.hashAlgo, accessor, + CanonPath { tmpDir + "/tmp" }).hash; + } + } + }(); ValidPathInfo newInfo0 { worker.store, @@ -2490,7 +2506,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() PosixSourceAccessor accessor; HashResult narHashAndSize = hashPath( accessor, CanonPath { actualPath }, - FileIngestionMethod::Recursive, HashAlgorithm::SHA256); + FileSerialisationMethod::Recursive, HashAlgorithm::SHA256); newInfo0.narHash = narHashAndSize.first; newInfo0.narSize = narHashAndSize.second; } @@ -2514,7 +2530,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() PosixSourceAccessor accessor; HashResult narHashAndSize = hashPath( accessor, CanonPath { actualPath }, - FileIngestionMethod::Recursive, HashAlgorithm::SHA256); + FileSerialisationMethod::Recursive, HashAlgorithm::SHA256); ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; newInfo0.narSize = narHashAndSize.second; auto refs = rewriteRefs(); diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 88152a645..f25cb9424 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -106,7 +106,7 @@ struct LocalDerivationGoal : public DerivationGoal RedirectedOutputs redirectedOutputs; /** - * The outputs paths used during the build. + * The output paths used during the build. * * - Input-addressed derivations or fixed content-addressed outputs are * sometimes built when some of their outputs already exist, and can not diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index d57e22393..815ded3d5 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -331,13 +331,23 @@ void Worker::run(const Goals & _topGoals) if (awake.empty() && 0U == settings.maxBuildJobs) { if (getMachines().empty()) - throw Error("unable to start any build; either increase '--max-jobs' " - "or enable remote builds." - "\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html"); + throw Error( + R"( + Unable to start any build; + either increase '--max-jobs' or enable remote builds. + + For more information run 'man nix.conf' and search for '/machines'. + )" + ); else - throw Error("unable to start any build; remote machines may not have " - "all required system features." - "\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html"); + throw Error( + R"( + Unable to start any build; + remote machines may not have all required system features. + + For more information run 'man nix.conf' and search for '/machines'. + )" + ); } assert(!awake.empty()); @@ -519,11 +529,11 @@ bool Worker::pathContentsGood(const StorePath & path) if (!pathExists(store.printStorePath(path))) res = false; else { - HashResult current = hashPath( + Hash current = hashPath( *store.getFSAccessor(), CanonPath { store.printStorePath(path) }, FileIngestionMethod::Recursive, info->narHash.algo); Hash nullHash(HashAlgorithm::SHA256); - res = info->narHash == nullHash || info->narHash == current.first; + res = info->narHash == nullHash || info->narHash == current; } pathContentsGoodCache.insert_or_assign(path, res); if (!res) diff --git a/src/libstore/builtins.hh b/src/libstore/builtins.hh index d201fb3ac..93558b49e 100644 --- a/src/libstore/builtins.hh +++ b/src/libstore/builtins.hh @@ -6,7 +6,13 @@ namespace nix { // TODO: make pluggable. -void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData); -void builtinUnpackChannel(const BasicDerivation & drv); +void builtinFetchurl( + const BasicDerivation & drv, + const std::map & outputs, + const std::string & netrcData); + +void builtinUnpackChannel( + const BasicDerivation & drv, + const std::map & outputs); } diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 9283251ac..1ed7b39cc 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -161,7 +161,9 @@ void buildProfile(const Path & out, Packages && pkgs) debug("created %d symlinks in user environment", state.symlinks); } -void builtinBuildenv(const BasicDerivation & drv) +void builtinBuildenv( + const BasicDerivation & drv, + const std::map & outputs) { auto getAttr = [&](const std::string & name) { auto i = drv.env.find(name); @@ -169,7 +171,7 @@ void builtinBuildenv(const BasicDerivation & drv) return i->second; }; - Path out = getAttr("out"); + auto out = outputs.at("out"); createDirs(out); /* Convert the stuff we get from the environment back into a diff --git a/src/libstore/builtins/buildenv.hh b/src/libstore/builtins/buildenv.hh index b24633e27..8e112e176 100644 --- a/src/libstore/builtins/buildenv.hh +++ b/src/libstore/builtins/buildenv.hh @@ -45,6 +45,8 @@ typedef std::vector Packages; void buildProfile(const Path & out, Packages && pkgs); -void builtinBuildenv(const BasicDerivation & drv); +void builtinBuildenv( + const BasicDerivation & drv, + const std::map & outputs); } diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 2086bd0b9..b9dfeba2f 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -6,7 +6,10 @@ namespace nix { -void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) +void builtinFetchurl( + const BasicDerivation & drv, + const std::map & outputs, + const std::string & netrcData) { /* Make the host's netrc data available. Too bad curl requires this to be stored in a file. It would be nice if we could just @@ -16,14 +19,15 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) writeFile(settings.netrcFile, netrcData, 0600); } - auto getAttr = [&](const std::string & name) { - auto i = drv.env.find(name); - if (i == drv.env.end()) throw Error("attribute '%s' missing", name); - return i->second; - }; + auto out = get(drv.outputs, "out"); + if (!out) + throw Error("'builtin:fetchurl' requires an 'out' output"); - Path storePath = getAttr("out"); - auto mainUrl = getAttr("url"); + if (!(drv.type().isFixed() || drv.type().isImpure())) + throw Error("'builtin:fetchurl' must be a fixed-output or impure derivation"); + + auto storePath = outputs.at("out"); + auto mainUrl = drv.env.at("url"); bool unpack = getOr(drv.env, "unpack", "") == "1"; /* Note: have to use a fresh fileTransfer here because we're in @@ -59,13 +63,12 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) }; /* Try the hashed mirrors first. */ - if (getAttr("outputHashMode") == "flat") + auto dof = std::get_if(&out->raw); + if (dof && dof->ca.method.getFileIngestionMethod() == FileIngestionMethod::Flat) for (auto hashedMirror : settings.hashedMirrors.get()) try { if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; - std::optional ht = parseHashAlgoOpt(getAttr("outputHashAlgo")); - Hash h = newHashAllowEmpty(getAttr("outputHash"), ht); - fetch(hashedMirror + printHashAlgo(h.algo) + "/" + h.to_string(HashFormat::Base16, false)); + fetch(hashedMirror + printHashAlgo(dof->ca.hash.algo) + "/" + dof->ca.hash.to_string(HashFormat::Base16, false)); return; } catch (Error & e) { debug(e.what()); diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc index ba04bb16c..6f68d4c0b 100644 --- a/src/libstore/builtins/unpack-channel.cc +++ b/src/libstore/builtins/unpack-channel.cc @@ -3,7 +3,9 @@ namespace nix { -void builtinUnpackChannel(const BasicDerivation & drv) +void builtinUnpackChannel( + const BasicDerivation & drv, + const std::map & outputs) { auto getAttr = [&](const std::string & name) { auto i = drv.env.find(name); @@ -11,7 +13,7 @@ void builtinUnpackChannel(const BasicDerivation & drv) return i->second; }; - Path out = getAttr("out"); + auto out = outputs.at("out"); auto channelName = getAttr("channelName"); auto src = getAttr("src"); diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index fc408f5af..4ed4f2de5 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -4,22 +4,44 @@ namespace nix { -std::string makeFileIngestionPrefix(FileIngestionMethod m) +std::string_view makeFileIngestionPrefix(FileIngestionMethod m) { switch (m) { case FileIngestionMethod::Flat: return ""; case FileIngestionMethod::Recursive: return "r:"; + case FileIngestionMethod::Git: + experimentalFeatureSettings.require(Xp::GitHashing); + return "git:"; default: throw Error("impossible, caught both cases"); } } -std::string ContentAddressMethod::renderPrefix() const +std::string_view ContentAddressMethod::render() const { return std::visit(overloaded { - [](TextIngestionMethod) -> std::string { return "text:"; }, + [](TextIngestionMethod) -> std::string_view { return "text"; }, + [](FileIngestionMethod m2) { + /* Not prefixed for back compat with things that couldn't produce text before. */ + return renderFileIngestionMethod(m2); + }, + }, raw); +} + +ContentAddressMethod ContentAddressMethod::parse(std::string_view m) +{ + if (m == "text") + return TextIngestionMethod {}; + else + return parseFileIngestionMethod(m); +} + +std::string_view ContentAddressMethod::renderPrefix() const +{ + return std::visit(overloaded { + [](TextIngestionMethod) -> std::string_view { return "text:"; }, [](FileIngestionMethod m2) { /* Not prefixed for back compat with things that couldn't produce text before. */ return makeFileIngestionPrefix(m2); @@ -32,13 +54,17 @@ ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m) if (splitPrefix(m, "r:")) { return FileIngestionMethod::Recursive; } + else if (splitPrefix(m, "git:")) { + experimentalFeatureSettings.require(Xp::GitHashing); + return FileIngestionMethod::Git; + } else if (splitPrefix(m, "text:")) { return TextIngestionMethod {}; } return FileIngestionMethod::Flat; } -std::string ContentAddressMethod::render(HashAlgorithm ha) const +std::string ContentAddressMethod::renderWithAlgo(HashAlgorithm ha) const { return std::visit(overloaded { [&](const TextIngestionMethod & th) { @@ -92,10 +118,10 @@ static std::pair parseContentAddressMethodP } auto parseHashAlgorithm_ = [&](){ - auto hashTypeRaw = splitPrefixTo(rest, ':'); - if (!hashTypeRaw) + auto hashAlgoRaw = splitPrefixTo(rest, ':'); + if (!hashAlgoRaw) throw UsageError("content address hash must be in form ':', but found: %s", wholeInput); - HashAlgorithm hashAlgo = parseHashAlgo(*hashTypeRaw); + HashAlgorithm hashAlgo = parseHashAlgo(*hashAlgoRaw); return hashAlgo; }; @@ -112,6 +138,10 @@ static std::pair parseContentAddressMethodP auto method = FileIngestionMethod::Flat; if (splitPrefix(rest, "r:")) method = FileIngestionMethod::Recursive; + else if (splitPrefix(rest, "git:")) { + experimentalFeatureSettings.require(Xp::GitHashing); + method = FileIngestionMethod::Git; + } HashAlgorithm hashAlgo = parseHashAlgorithm_(); return { std::move(method), @@ -133,7 +163,7 @@ ContentAddress ContentAddress::parse(std::string_view rawCa) }; } -std::pair ContentAddressMethod::parse(std::string_view caMethod) +std::pair ContentAddressMethod::parseWithAlgo(std::string_view caMethod) { std::string asPrefix = std::string{caMethod} + ":"; // parseContentAddressMethodPrefix takes its argument by reference @@ -155,7 +185,7 @@ std::string renderContentAddress(std::optional ca) std::string ContentAddress::printMethodAlgo() const { - return method.renderPrefix() + return std::string { method.renderPrefix() } + printHashAlgo(hash.algo); } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index f0973412b..5925f8e01 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -36,7 +36,7 @@ struct TextIngestionMethod : std::monostate { }; * Compute the prefix to the hash algorithm which indicates how the * files were ingested. */ -std::string makeFileIngestionPrefix(FileIngestionMethod m); +std::string_view makeFileIngestionPrefix(FileIngestionMethod m); /** * An enumeration of all the ways we can content-address store objects. @@ -59,6 +59,20 @@ struct ContentAddressMethod MAKE_WRAPPER_CONSTRUCTOR(ContentAddressMethod); + /** + * Parse a content addressing method (name). + * + * The inverse of `render`. + */ + static ContentAddressMethod parse(std::string_view rawCaMethod); + + /** + * Render a content addressing method (name). + * + * The inverse of `parse`. + */ + std::string_view render() const; + /** * Parse the prefix tag which indicates how the files * were ingested, with the fixed output case not prefixed for back @@ -74,20 +88,20 @@ struct ContentAddressMethod * * The rough inverse of `parsePrefix()`. */ - std::string renderPrefix() const; + std::string_view renderPrefix() const; /** - * Parse a content addressing method and hash type. + * Parse a content addressing method and hash algorithm. */ - static std::pair parse(std::string_view rawCaMethod); + static std::pair parseWithAlgo(std::string_view rawCaMethod); /** - * Render a content addressing method and hash type in a + * Render a content addressing method and hash algorithm in a * nicer way, prefixing both cases. * * The rough inverse of `parse()`. */ - std::string render(HashAlgorithm ht) const; + std::string renderWithAlgo(HashAlgorithm ha) const; /** * Get the underlying way to content-address file system objects. @@ -113,7 +127,7 @@ struct ContentAddressMethod * ‘text:sha256:’ * * - `FixedIngestionMethod`: - * ‘fixed:::’ + * ‘fixed:::’ */ struct ContentAddress { diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 27ad14ed4..e1337f51d 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -13,6 +13,7 @@ #include "archive.hh" #include "derivations.hh" #include "args.hh" +#include "git.hh" namespace nix::daemon { @@ -119,7 +120,7 @@ struct TunnelLogger : public Logger if (GET_PROTOCOL_MINOR(clientVersion) >= 26) { to << STDERR_ERROR << *ex; } else { - to << STDERR_ERROR << ex->what() << ex->status; + to << STDERR_ERROR << ex->what() << ex->info().status; } } } @@ -400,11 +401,23 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto pathInfo = [&]() { // NB: FramedSource must be out of scope before logger->stopWork(); - auto [contentAddressMethod, hashAlgo_] = ContentAddressMethod::parse(camStr); - auto hashAlgo = hashAlgo_; // work around clang bug + auto [contentAddressMethod, hashAlgo] = ContentAddressMethod::parseWithAlgo(camStr); FramedSource source(from); + FileSerialisationMethod dumpMethod; + switch (contentAddressMethod.getFileIngestionMethod()) { + case FileIngestionMethod::Flat: + dumpMethod = FileSerialisationMethod::Flat; + break; + case FileIngestionMethod::Recursive: + dumpMethod = FileSerialisationMethod::Recursive; + break; + case FileIngestionMethod::Git: + // Use NAR; Git is not a serialization method + dumpMethod = FileSerialisationMethod::Recursive; + break; + } // TODO these two steps are essentially RemoteStore::addCAToStore. Move it up to Store. - auto path = store->addToStoreFromDump(source, name, contentAddressMethod, hashAlgo, refs, repair); + auto path = store->addToStoreFromDump(source, name, dumpMethod, contentAddressMethod, hashAlgo, refs, repair); return store->queryPathInfo(path); }(); logger->stopWork(); @@ -430,30 +443,23 @@ static void performOp(TunnelLogger * logger, ref store, hashAlgo = parseHashAlgo(hashAlgoRaw); } + // Old protocol always sends NAR, regardless of hashing method auto dumpSource = sinkToSource([&](Sink & saved) { - if (method == FileIngestionMethod::Recursive) { - /* We parse the NAR dump through into `saved` unmodified, - so why all this extra work? We still parse the NAR so - that we aren't sending arbitrary data to `saved` - unwittingly`, and we know when the NAR ends so we don't - consume the rest of `from` and can't parse another - command. (We don't trust `addToStoreFromDump` to not - eagerly consume the entire stream it's given, past the - length of the Nar. */ - TeeSource savedNARSource(from, saved); - NullFileSystemObjectSink sink; /* just parse the NAR */ - parseDump(sink, savedNARSource); - } else { - /* Incrementally parse the NAR file, stripping the - metadata, and streaming the sole file we expect into - `saved`. */ - RegularFileSink savedRegular { saved }; - parseDump(savedRegular, from); - if (!savedRegular.regular) throw Error("regular file expected"); - } + /* We parse the NAR dump through into `saved` unmodified, + so why all this extra work? We still parse the NAR so + that we aren't sending arbitrary data to `saved` + unwittingly`, and we know when the NAR ends so we don't + consume the rest of `from` and can't parse another + command. (We don't trust `addToStoreFromDump` to not + eagerly consume the entire stream it's given, past the + length of the Nar. */ + TeeSource savedNARSource(from, saved); + NullFileSystemObjectSink sink; /* just parse the NAR */ + parseDump(sink, savedNARSource); }); logger->startWork(); - auto path = store->addToStoreFromDump(*dumpSource, baseName, method, hashAlgo); + auto path = store->addToStoreFromDump( + *dumpSource, baseName, FileSerialisationMethod::Recursive, method, hashAlgo); logger->stopWork(); to << store->printStorePath(path); @@ -485,7 +491,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto path = ({ StringSource source { s }; - store->addToStoreFromDump(source, suffix, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, NoRepair); + store->addToStoreFromDump(source, suffix, FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, NoRepair); }); logger->stopWork(); to << store->printStorePath(path); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 393806652..df14e979f 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -150,7 +150,7 @@ StorePath writeDerivation(Store & store, }) : ({ StringSource s { contents }; - store.addToStoreFromDump(s, suffix, TextIngestionMethod {}, HashAlgorithm::SHA256, references, repair); + store.addToStoreFromDump(s, suffix, FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, references, repair); }); } @@ -601,7 +601,7 @@ std::string Derivation::unparse(const StoreDirConfig & store, bool maskOutputs, }, [&](const DerivationOutput::CAFloating & dof) { s += ','; printUnquotedString(s, ""); - s += ','; printUnquotedString(s, dof.method.renderPrefix() + printHashAlgo(dof.hashAlgo)); + s += ','; printUnquotedString(s, std::string { dof.method.renderPrefix() } + printHashAlgo(dof.hashAlgo)); s += ','; printUnquotedString(s, ""); }, [&](const DerivationOutput::Deferred &) { @@ -612,7 +612,7 @@ std::string Derivation::unparse(const StoreDirConfig & store, bool maskOutputs, [&](const DerivationOutput::Impure & doi) { // FIXME s += ','; printUnquotedString(s, ""); - s += ','; printUnquotedString(s, doi.method.renderPrefix() + printHashAlgo(doi.hashAlgo)); + s += ','; printUnquotedString(s, std::string { doi.method.renderPrefix() } + printHashAlgo(doi.hashAlgo)); s += ','; printUnquotedString(s, "impure"); } }, i.second.raw); @@ -701,7 +701,7 @@ DerivationType BasicDerivation::type() const floatingHashAlgo = dof.hashAlgo; } else { if (*floatingHashAlgo != dof.hashAlgo) - throw Error("all floating outputs must use the same hash type"); + throw Error("all floating outputs must use the same hash algorithm"); } }, [&](const DerivationOutput::Deferred &) { @@ -984,7 +984,7 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva }, [&](const DerivationOutput::CAFloating & dof) { out << "" - << (dof.method.renderPrefix() + printHashAlgo(dof.hashAlgo)) + << (std::string { dof.method.renderPrefix() } + printHashAlgo(dof.hashAlgo)) << ""; }, [&](const DerivationOutput::Deferred &) { @@ -994,7 +994,7 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva }, [&](const DerivationOutput::Impure & doi) { out << "" - << (doi.method.renderPrefix() + printHashAlgo(doi.hashAlgo)) + << (std::string { doi.method.renderPrefix() } + printHashAlgo(doi.hashAlgo)) << "impure"; }, }, i.second.raw); @@ -1221,11 +1221,11 @@ nlohmann::json DerivationOutput::toJSON( // FIXME print refs? }, [&](const DerivationOutput::CAFloating & dof) { - res["hashAlgo"] = dof.method.renderPrefix() + printHashAlgo(dof.hashAlgo); + res["hashAlgo"] = std::string { dof.method.renderPrefix() } + printHashAlgo(dof.hashAlgo); }, [&](const DerivationOutput::Deferred &) {}, [&](const DerivationOutput::Impure & doi) { - res["hashAlgo"] = doi.method.renderPrefix() + printHashAlgo(doi.hashAlgo); + res["hashAlgo"] = std::string { doi.method.renderPrefix() } + printHashAlgo(doi.hashAlgo); res["impure"] = true; }, }, raw); diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index e4f13b8f4..30f23cff9 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -61,7 +61,8 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store virtual StorePath addToStoreFromDump( Source & dump, std::string_view name, - ContentAddressMethod method = FileIngestionMethod::Recursive, + FileSerialisationMethod dumpMethod = FileSerialisationMethod::Recursive, + ContentAddressMethod hashMethod = FileIngestionMethod::Recursive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), RepairFlag repair = NoRepair) override diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index dcbec4acd..bab21bf51 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -106,6 +106,8 @@ struct curlFileTransfer : public FileTransfer this->result.data.append(data); }) { + result.urls.push_back(request.uri); + requestHeaders = curl_slist_append(requestHeaders, "Accept-Encoding: zstd, br, gzip, deflate, bzip2, xz"); if (!request.expectedETag.empty()) requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str()); @@ -182,6 +184,14 @@ struct curlFileTransfer : public FileTransfer return ((TransferItem *) userp)->writeCallback(contents, size, nmemb); } + void appendCurrentUrl() + { + char * effectiveUriCStr = nullptr; + curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr); + if (effectiveUriCStr && *result.urls.rbegin() != effectiveUriCStr) + result.urls.push_back(effectiveUriCStr); + } + size_t headerCallback(void * contents, size_t size, size_t nmemb) { size_t realSize = size * nmemb; @@ -196,6 +206,7 @@ struct curlFileTransfer : public FileTransfer statusMsg = trim(match.str(1)); acceptRanges = false; encoding = ""; + appendCurrentUrl(); } else { auto i = line.find(':'); @@ -360,14 +371,11 @@ struct curlFileTransfer : public FileTransfer { auto httpStatus = getHTTPStatus(); - char * effectiveUriCStr = nullptr; - curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr); - if (effectiveUriCStr) - result.effectiveUri = effectiveUriCStr; - debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes", request.verb(), request.uri, code, httpStatus, result.bodySize); + appendCurrentUrl(); + if (decompressionSink) { try { decompressionSink->finish(); @@ -779,7 +787,10 @@ FileTransferResult FileTransfer::upload(const FileTransferRequest & request) return enqueueFileTransfer(request).get(); } -void FileTransfer::download(FileTransferRequest && request, Sink & sink) +void FileTransfer::download( + FileTransferRequest && request, + Sink & sink, + std::function resultCallback) { /* Note: we can't call 'sink' via request.dataCallback, because that would cause the sink to execute on the fileTransfer @@ -829,11 +840,13 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink) }; enqueueFileTransfer(request, - {[_state](std::future fut) { + {[_state, resultCallback{std::move(resultCallback)}](std::future fut) { auto state(_state->lock()); state->quit = true; try { - fut.get(); + auto res = fut.get(); + if (resultCallback) + resultCallback(std::move(res)); } catch (...) { state->exc = std::current_exception(); } @@ -882,12 +895,12 @@ template FileTransferError::FileTransferError(FileTransfer::Error error, std::optional response, const Args & ... args) : Error(args...), error(error), response(response) { - const auto hf = hintfmt(args...); + const auto hf = HintFmt(args...); // FIXME: Due to https://github.com/NixOS/nix/issues/3841 we don't know how // to print different messages for different verbosity levels. For now // we add some heuristics for detecting when we want to show the response. if (response && (response->size() < 1024 || response->find("") != std::string::npos)) - err.msg = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), chomp(*response)); + err.msg = HintFmt("%1%\n\nresponse body:\n\n%2%", Uncolored(hf.str()), chomp(*response)); else err.msg = hf; } diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index a3b0dde1f..1c271cbec 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -75,14 +75,34 @@ struct FileTransferRequest struct FileTransferResult { + /** + * Whether this is a cache hit (i.e. the ETag supplied in the + * request is still valid). If so, `data` is empty. + */ bool cached = false; + + /** + * The ETag of the object. + */ std::string etag; - std::string effectiveUri; + + /** + * All URLs visited in the redirect chain. + */ + std::vector urls; + + /** + * The response body. + */ std::string data; + uint64_t bodySize = 0; - /* An "immutable" URL for this resource (i.e. one whose contents - will never change), as returned by the `Link: ; - rel="immutable"` header. */ + + /** + * An "immutable" URL for this resource (i.e. one whose contents + * will never change), as returned by the `Link: ; + * rel="immutable"` header. + */ std::optional immutableUrl; }; @@ -116,7 +136,10 @@ struct FileTransfer * Download a file, writing its data to a sink. The sink will be * invoked on the thread of the caller. */ - void download(FileTransferRequest && request, Sink & sink); + void download( + FileTransferRequest && request, + Sink & sink, + std::function resultCallback = {}); enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; }; diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 070e252b6..e6544976a 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -180,14 +180,21 @@ public: getDefaultCores(), "cores", R"( - Sets the value of the `NIX_BUILD_CORES` environment variable in the - invocation of builders. Builders can use this variable at their - discretion to control the maximum amount of parallelism. For - instance, in Nixpkgs, if the derivation attribute - `enableParallelBuilding` is set to `true`, the builder passes the - `-jN` flag to GNU Make. It can be overridden using the `--cores` - command line switch and defaults to `1`. The value `0` means that - the builder should use all available CPU cores in the system. + Sets the value of the `NIX_BUILD_CORES` environment variable in the [invocation of the `builder` executable](@docroot@/language/derivations.md#builder-execution) of a derivation. + The `builder` executable can use this variable to control its own maximum amount of parallelism. + + + For instance, in Nixpkgs, if the attribute `enableParallelBuilding` for the `mkDerivation` build helper is set to `true`, it will pass the `-j${NIX_BUILD_CORES}` flag to GNU Make. + + The value `0` means that the `builder` should use all available CPU cores in the system. + + > **Note** + > + > The number of parallel local Nix build jobs is independently controlled with the [`max-jobs`](#conf-max-jobs) setting. )", {"build-cores"}, // Don't document the machine-specific default value @@ -270,9 +277,121 @@ public: Setting builders{ this, "@" + nixConfDir + "/machines", "builders", R"( - A semicolon-separated list of build machines. - For the exact format and examples, see [the manual chapter on remote builds](../advanced-topics/distributed-builds.md) - )"}; + A semicolon- or newline-separated list of build machines. + + In addition to the [usual ways of setting configuration options](@docroot@/command-ref/conf-file.md), the value can be read from a file by prefixing its absolute path with `@`. + + > **Example** + > + > This is the default setting: + > + > ``` + > builders = @/etc/nix/machines + > ``` + + Each machine specification consists of the following elements, separated by spaces. + Only the first element is required. + To leave a field at its default, set it to `-`. + + 1. The URI of the remote store in the format `ssh://[username@]hostname`. + + > **Example** + > + > `ssh://nix@mac` + + For backward compatibility, `ssh://` may be omitted. + The hostname may be an alias defined in `~/.ssh/config`. + + 2. A comma-separated list of [Nix system types](@docroot@/contributing/hacking.md#system-type). + If omitted, this defaults to the local platform type. + + > **Example** + > + > `aarch64-darwin` + + It is possible for a machine to support multiple platform types. + + > **Example** + > + > `i686-linux,x86_64-linux` + + 3. The SSH identity file to be used to log in to the remote machine. + If omitted, SSH will use its regular identities. + + > **Example** + > + > `/home/user/.ssh/id_mac` + + 4. The maximum number of builds that Nix will execute in parallel on the machine. + Typically this should be equal to the number of CPU cores. + + 5. The “speed factor”, indicating the relative speed of the machine as a positive integer. + If there are multiple machines of the right type, Nix will prefer the fastest, taking load into account. + + 6. A comma-separated list of supported [system features](#conf-system-features). + + A machine will only be used to build a derivation if all the features in the derivation's [`requiredSystemFeatures`](@docroot@/language/advanced-attributes.html#adv-attr-requiredSystemFeatures) attribute are supported by that machine. + + 7. A comma-separated list of required [system features](#conf-system-features). + + A machine will only be used to build a derivation if all of the machine’s required features appear in the derivation’s [`requiredSystemFeatures`](@docroot@/language/advanced-attributes.html#adv-attr-requiredSystemFeatures) attribute. + + 8. The (base64-encoded) public host key of the remote machine. + If omitted, SSH will use its regular `known_hosts` file. + + The value for this field can be obtained via `base64 -w0`. + + > **Example** + > + > Multiple builders specified on the command line: + > + > ```console + > --builders 'ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd' + > ``` + + > **Example** + > + > This specifies several machines that can perform `i686-linux` builds: + > + > ``` + > nix@scratchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy 8 1 kvm + > nix@itchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy 8 2 + > nix@poochie.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy 1 2 kvm benchmark + > ``` + > + > However, `poochie` will only build derivations that have the attribute + > + > ```nix + > requiredSystemFeatures = [ "benchmark" ]; + > ``` + > + > or + > + > ```nix + > requiredSystemFeatures = [ "benchmark" "kvm" ]; + > ``` + > + > `itchy` cannot do builds that require `kvm`, but `scratchy` does support such builds. + > For regular builds, `itchy` will be preferred over `scratchy` because it has a higher speed factor. + + For Nix to use substituters, the calling user must be in the [`trusted-users`](#conf-trusted-users) list. + + > **Note** + > + > A build machine must be accessible via SSH and have Nix installed. + > `nix` must be available in `$PATH` for the user connecting over SSH. + + > **Warning** + > + > If you are building via the Nix daemon (default), the Nix daemon user account on the local machine (that is, `root`) requires access to a user account on the remote machine (not necessarily `root`). + > + > If you can’t or don’t want to configure `root` to be able to access the remote machine, set [`store`](#conf-store) to any [local store](@docroot@/store/types/local-store.html), e.g. by passing `--store /tmp` to the command on the local machine. + + To build only on remote machines and disable local builds, set [`max-jobs`](#conf-max-jobs) to 0. + + If you want the remote machines to use substituters, set [`builders-use-substitutes`](#conf-builders-use-substituters) to `true`. + )", + {}, false}; Setting alwaysAllowSubstitutes{ this, false, "always-allow-substitutes", @@ -793,10 +912,17 @@ public: Setting ttlNegativeNarInfoCache{ this, 3600, "narinfo-cache-negative-ttl", R"( - The TTL in seconds for negative lookups. If a store path is queried - from a substituter but was not found, there will be a negative - lookup cached in the local disk cache database for the specified - duration. + The TTL in seconds for negative lookups. + If a store path is queried from a [substituter](#conf-substituters) but was not found, there will be a negative lookup cached in the local disk cache database for the specified duration. + + Set to `0` to force updating the lookup cache. + + To wipe the lookup cache completely: + + ```shell-session + $ rm $HOME/.cache/nix/binary-cache-v*.sqlite* + # rm /root/.cache/nix/binary-cache-v*.sqlite* + ``` )"}; Setting ttlPositiveNarInfoCache{ @@ -968,8 +1094,8 @@ public: this, {}, "hashed-mirrors", R"( A list of web servers used by `builtins.fetchurl` to obtain files by - hash. Given a hash type *ht* and a base-16 hash *h*, Nix will try to - download the file from *hashed-mirror*/*ht*/*h*. This allows files to + hash. Given a hash algorithm *ha* and a base-16 hash *h*, Nix will try to + download the file from *hashed-mirror*/*ha*/*h*. This allows files to be downloaded even if they have disappeared from their original URI. For example, given an example mirror `http://tarballs.nixos.org/`, when building the derivation diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh index ae890177b..ca2f115d2 100644 --- a/src/libstore/legacy-ssh-store.hh +++ b/src/libstore/legacy-ssh-store.hh @@ -72,7 +72,8 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor virtual StorePath addToStoreFromDump( Source & dump, std::string_view name, - ContentAddressMethod method = FileIngestionMethod::Recursive, + FileSerialisationMethod dumpMethod = FileSerialisationMethod::Recursive, + ContentAddressMethod hashMethod = FileIngestionMethod::Recursive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), RepairFlag repair = NoRepair) override diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 953f3a264..81c385ddb 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -28,7 +28,7 @@ struct LocalStoreAccessor : PosixSourceAccessor auto [storePath, rest] = store->toStorePath(path.abs()); if (requireValidPath && !store->isValidPath(storePath)) throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); - return CanonPath(store->getRealStoreDir()) + storePath.to_string() + CanonPath(rest); + return CanonPath(store->getRealStoreDir()) / storePath.to_string() / CanonPath(rest); } std::optional maybeLstat(const CanonPath & path) override diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index bf855b67e..8fb081200 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -43,7 +43,7 @@ public: LocalFSStore(const Params & params); void narFromPath(const StorePath & path, Sink & sink) override; - ref getFSAccessor(bool requireValidPath) override; + ref getFSAccessor(bool requireValidPath = true) override; /** * Creates symlink from the `gcRoot` to the `storePath` and diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 7c40fa790..a7c34ab4a 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1,5 +1,6 @@ #include "local-store.hh" #include "globals.hh" +#include "git.hh" #include "archive.hh" #include "pathlocks.hh" #include "worker-protocol.hh" @@ -1103,19 +1104,29 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (info.ca) { auto & specified = *info.ca; auto actualHash = ({ - HashModuloSink caSink { - specified.hash.algo, - std::string { info.path.hashPart() }, - }; - PosixSourceAccessor accessor; - dumpPath( - *getFSAccessor(false), - CanonPath { printStorePath(info.path) }, - caSink, - specified.method.getFileIngestionMethod()); + auto accessor = getFSAccessor(false); + CanonPath path { printStorePath(info.path) }; + Hash h { HashAlgorithm::SHA256 }; // throwaway def to appease C++ + auto fim = specified.method.getFileIngestionMethod(); + switch (fim) { + case FileIngestionMethod::Flat: + case FileIngestionMethod::Recursive: + { + HashModuloSink caSink { + specified.hash.algo, + std::string { info.path.hashPart() }, + }; + dumpPath(*accessor, path, caSink, (FileSerialisationMethod) fim); + h = caSink.finish().first; + break; + } + case FileIngestionMethod::Git: + h = git::dumpHash(specified.hash.algo, *accessor, path).hash; + break; + } ContentAddress { .method = specified.method, - .hash = caSink.finish().first, + .hash = std::move(h), }; }); if (specified.hash != actualHash.hash) { @@ -1143,7 +1154,8 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, StorePath LocalStore::addToStoreFromDump( Source & source0, std::string_view name, - ContentAddressMethod method, + FileSerialisationMethod dumpMethod, + ContentAddressMethod hashMethod, HashAlgorithm hashAlgo, const StorePathSet & references, RepairFlag repair) @@ -1196,7 +1208,13 @@ StorePath LocalStore::addToStoreFromDump( Path tempDir; AutoCloseFD tempDirFd; - if (!inMemory) { + bool methodsMatch = ContentAddressMethod(FileIngestionMethod(dumpMethod)) == hashMethod; + + /* If the methods don't match, our streaming hash of the dump is the + wrong sort, and we need to rehash. */ + bool inMemoryAndDontNeedRestore = inMemory && methodsMatch; + + if (!inMemoryAndDontNeedRestore) { /* Drain what we pulled so far, and then keep on pulling */ StringSource dumpSource { dump }; ChainSource bothSource { dumpSource, source }; @@ -1205,17 +1223,23 @@ StorePath LocalStore::addToStoreFromDump( delTempDir = std::make_unique(tempDir); tempPath = tempDir + "/x"; - restorePath(tempPath, bothSource, method.getFileIngestionMethod()); + restorePath(tempPath, bothSource, dumpMethod); dumpBuffer.reset(); dump = {}; } - auto [hash, size] = hashSink->finish(); + auto [dumpHash, size] = hashSink->finish(); + + PosixSourceAccessor accessor; auto desc = ContentAddressWithReferences::fromParts( - method, - hash, + hashMethod, + methodsMatch + ? dumpHash + : hashPath( + accessor, CanonPath { tempPath }, + hashMethod.getFileIngestionMethod(), hashAlgo), { .others = references, // caller is not capable of creating a self-reference, because this is content-addressed without modulus @@ -1241,10 +1265,20 @@ StorePath LocalStore::addToStoreFromDump( autoGC(); - if (inMemory) { + if (inMemoryAndDontNeedRestore) { StringSource dumpSource { dump }; /* Restore from the buffer in memory. */ - restorePath(realPath, dumpSource, method.getFileIngestionMethod()); + auto fim = hashMethod.getFileIngestionMethod(); + switch (fim) { + case FileIngestionMethod::Flat: + case FileIngestionMethod::Recursive: + restorePath(realPath, dumpSource, (FileSerialisationMethod) fim); + break; + case FileIngestionMethod::Git: + // doesn't correspond to serialization method, so + // this should be unreachable + assert(false); + } } else { /* Move the temporary path we restored above. */ moveFile(tempPath, realPath); @@ -1252,8 +1286,8 @@ StorePath LocalStore::addToStoreFromDump( /* For computing the nar hash. In recursive SHA-256 mode, this is the same as the store hash, so no need to do it again. */ - auto narHash = std::pair { hash, size }; - if (method != FileIngestionMethod::Recursive || hashAlgo != HashAlgorithm::SHA256) { + auto narHash = std::pair { dumpHash, size }; + if (dumpMethod != FileSerialisationMethod::Recursive || hashAlgo != HashAlgorithm::SHA256) { HashSink narSink { HashAlgorithm::SHA256 }; dumpPath(realPath, narSink); narHash = narSink.finish(); @@ -1345,7 +1379,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) PosixSourceAccessor accessor; std::string hash = hashPath( accessor, CanonPath { linkPath }, - FileIngestionMethod::Recursive, HashAlgorithm::SHA256).first.to_string(HashFormat::Nix32, false); + FileIngestionMethod::Recursive, HashAlgorithm::SHA256).to_string(HashFormat::Nix32, false); if (hash != link.name) { printError("link '%s' was modified! expected hash '%s', got '%s'", linkPath, link.name, hash); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 11b8f5814..47d3c04bc 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -180,7 +180,8 @@ public: StorePath addToStoreFromDump( Source & dump, std::string_view name, - ContentAddressMethod method, + FileSerialisationMethod dumpMethod, + ContentAddressMethod hashMethod, HashAlgorithm hashAlgo, const StorePathSet & references, RepairFlag repair) override; diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index b13e4c52c..cecf8148f 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -277,7 +277,7 @@ json listNar(ref accessor, const CanonPath & path, bool recurse) json &res2 = obj["entries"]; for (const auto & [name, type] : accessor->readDirectory(path)) { if (recurse) { - res2[name] = listNar(accessor, path + name, true); + res2[name] = listNar(accessor, path / name, true); } else res2[name] = json::object(); } diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 310105c75..07beb8acb 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -209,7 +209,7 @@ public: { auto r(state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority)); - assert(r.next()); + if (!r.next()) { abort(); } ret.id = (int) r.getInt(0); } diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 78e4f6d86..daaaaf073 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -151,7 +151,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, PosixSourceAccessor accessor; hashPath( accessor, CanonPath { path }, - FileIngestionMethod::Recursive, HashAlgorithm::SHA256).first; + FileSerialisationMethod::Recursive, HashAlgorithm::SHA256).first; }); debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Nix32, true)); @@ -166,7 +166,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, PosixSourceAccessor accessor; hashPath( accessor, CanonPath { linkPath }, - FileIngestionMethod::Recursive, HashAlgorithm::SHA256).first; + FileSerialisationMethod::Recursive, HashAlgorithm::SHA256).first; }))) { // XXX: Consider overwriting linkPath with our valid version. diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index ccf95beef..8dfe8adda 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -13,6 +13,7 @@ #include "derivations.hh" #include "pool.hh" #include "finally.hh" +#include "git.hh" #include "logging.hh" #include "callback.hh" #include "filetransfer.hh" @@ -435,7 +436,7 @@ ref RemoteStore::addCAToStore( conn->to << WorkerProto::Op::AddToStore << name - << caMethod.render(hashAlgo); + << caMethod.renderWithAlgo(hashAlgo); WorkerProto::write(*this, *conn, references); conn->to << repair; @@ -508,12 +509,28 @@ ref RemoteStore::addCAToStore( StorePath RemoteStore::addToStoreFromDump( Source & dump, std::string_view name, - ContentAddressMethod method, + FileSerialisationMethod dumpMethod, + ContentAddressMethod hashMethod, HashAlgorithm hashAlgo, const StorePathSet & references, RepairFlag repair) { - return addCAToStore(dump, name, method, hashAlgo, references, repair)->path; + FileSerialisationMethod fsm; + switch (hashMethod.getFileIngestionMethod()) { + case FileIngestionMethod::Flat: + fsm = FileSerialisationMethod::Flat; + break; + case FileIngestionMethod::Recursive: + fsm = FileSerialisationMethod::Recursive; + break; + case FileIngestionMethod::Git: + // Use NAR; Git is not a serialization method + fsm = FileSerialisationMethod::Recursive; + break; + } + if (fsm != dumpMethod) + unsupported("RemoteStore::addToStoreFromDump doesn't support this `dumpMethod` `hashMethod` combination"); + return addCAToStore(dump, name, hashMethod, hashAlgo, references, repair)->path; } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 87704985b..d630adc08 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -87,7 +87,8 @@ public: StorePath addToStoreFromDump( Source & dump, std::string_view name, - ContentAddressMethod method = FileIngestionMethod::Recursive, + FileSerialisationMethod dumpMethod = FileSerialisationMethod::Recursive, + ContentAddressMethod hashMethod = FileIngestionMethod::Recursive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), RepairFlag repair = NoRepair) override; @@ -184,7 +185,7 @@ protected: friend struct ConnectionHandle; - virtual ref getFSAccessor(bool requireValidPath) override; + virtual ref getFSAccessor(bool requireValidPath = true) override; virtual void narFromPath(const StorePath & path, Sink & sink) override; diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index d7432a305..06abfb90b 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -10,19 +10,19 @@ namespace nix { -SQLiteError::SQLiteError(const char *path, const char *errMsg, int errNo, int extendedErrNo, int offset, hintformat && hf) +SQLiteError::SQLiteError(const char *path, const char *errMsg, int errNo, int extendedErrNo, int offset, HintFmt && hf) : Error(""), path(path), errMsg(errMsg), errNo(errNo), extendedErrNo(extendedErrNo), offset(offset) { auto offsetStr = (offset == -1) ? "" : "at offset " + std::to_string(offset) + ": "; - err.msg = hintfmt("%s: %s%s, %s (in '%s')", - normaltxt(hf.str()), + err.msg = HintFmt("%s: %s%s, %s (in '%s')", + Uncolored(hf.str()), offsetStr, sqlite3_errstr(extendedErrNo), errMsg, path ? path : "(in-memory)"); } -[[noreturn]] void SQLiteError::throw_(sqlite3 * db, hintformat && hf) +[[noreturn]] void SQLiteError::throw_(sqlite3 * db, HintFmt && hf) { int err = sqlite3_errcode(db); int exterr = sqlite3_extended_errcode(db); @@ -33,7 +33,7 @@ SQLiteError::SQLiteError(const char *path, const char *errMsg, int errNo, int ex if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) { auto exp = SQLiteBusy(path, errMsg, err, exterr, offset, std::move(hf)); - exp.err.msg = hintfmt( + exp.err.msg = HintFmt( err == SQLITE_PROTOCOL ? "SQLite database '%s' is busy (SQLITE_PROTOCOL)" : "SQLite database '%s' is busy", @@ -249,7 +249,7 @@ void handleSQLiteBusy(const SQLiteBusy & e, time_t & nextWarning) if (now > nextWarning) { nextWarning = now + 10; logWarning({ - .msg = hintfmt(e.what()) + .msg = HintFmt(e.what()) }); } diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh index 0c08267f7..003e4d101 100644 --- a/src/libstore/sqlite.hh +++ b/src/libstore/sqlite.hh @@ -142,19 +142,19 @@ struct SQLiteError : Error template [[noreturn]] static void throw_(sqlite3 * db, const std::string & fs, const Args & ... args) { - throw_(db, hintfmt(fs, args...)); + throw_(db, HintFmt(fs, args...)); } - SQLiteError(const char *path, const char *errMsg, int errNo, int extendedErrNo, int offset, hintformat && hf); + SQLiteError(const char *path, const char *errMsg, int errNo, int extendedErrNo, int offset, HintFmt && hf); protected: template SQLiteError(const char *path, const char *errMsg, int errNo, int extendedErrNo, int offset, const std::string & fs, const Args & ... args) - : SQLiteError(path, errNo, extendedErrNo, offset, hintfmt(fs, args...)) + : SQLiteError(path, errMsg, errNo, extendedErrNo, offset, HintFmt(fs, args...)) { } - [[noreturn]] static void throw_(sqlite3 * db, hintformat && hf); + [[noreturn]] static void throw_(sqlite3 * db, HintFmt && hf); }; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 7b4175a92..4e8ba6f6f 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -12,7 +12,9 @@ #include "references.hh" #include "archive.hh" #include "callback.hh" +#include "git.hh" #include "remote-store.hh" +#include "posix-source-accessor.hh" // FIXME this should not be here, see TODO below on // `addMultipleToStore`. #include "worker-protocol.hh" @@ -65,85 +67,13 @@ StorePath Store::followLinksToStorePath(std::string_view path) const } -/* Store paths have the following form: +/* +The exact specification of store paths is in `protocols/store-path.md` +in the Nix manual. These few functions implement that specification. - = /- - - where - - = the location of the Nix store, usually /nix/store - - = a human readable name for the path, typically obtained - from the name attribute of the derivation, or the name of the - source file from which the store path is created. For derivation - outputs other than the default "out" output, the string "-" - is suffixed to . - - = base-32 representation of the first 160 bits of a SHA-256 - hash of ; the hash part of the store name - - = the string ":sha256:

::"; - note that it includes the location of the store as well as the - name to make sure that changes to either of those are reflected - in the hash (e.g. you won't get /nix/store/-name1 and - /nix/store/-name2 with equal hash parts). - - = one of: - "text:::..." - for plain text files written to the store using - addTextToStore(); ... are the store paths referenced - by this path, in the form described by - "source:::...::self" - for paths copied to the store using addToStore() when recursive - = true and hashAlgo = "sha256". Just like in the text case, we - can have the store paths referenced by the path. - Additionally, we can have an optional :self label to denote self - reference. - "output:" - for either the outputs created by derivations, OR paths copied - to the store using addToStore() with recursive != true or - hashAlgo != "sha256" (in that case "source" is used; it's - silly, but it's done that way for compatibility). is the - name of the output (usually, "out"). - -

= base-16 representation of a SHA-256 hash of - - = - if = "text:...": - the string written to the resulting store path - if = "source:...": - the serialisation of the path from which this store path is - copied, as returned by hashPath() - if = "output:": - for non-fixed derivation outputs: - the derivation (see hashDerivationModulo() in - primops.cc) - for paths copied by addToStore() or produced by fixed-output - derivations: - the string "fixed:out:::", where - = "r:" for recursive (path) hashes, or "" for flat - (file) hashes - = "md5", "sha1" or "sha256" - = base-16 representation of the path or flat hash of - the contents of the path (or expected contents of the - path for fixed-output derivations) - - Note that since an output derivation has always type output, while - something added by addToStore can have type output or source depending - on the hash, this means that the same input can be hashed differently - if added to the store via addToStore or via a derivation, in the sha256 - recursive case. - - It would have been nicer to handle fixed-output derivations under - "source", e.g. have something like "source:", but we're - stuck with this for now... - - The main reason for this way of computing names is to prevent name - collisions (for security). For instance, it shouldn't be feasible - to come up with a derivation whose output path collides with the - path for a copied source. The former would have a starting with - "output:out:", while the latter would have a starting with - "source:". +If changes to these functions go beyond mere implementation changes i.e. +also update the user-visible behavior, please update the specification +to match. */ @@ -191,6 +121,9 @@ static std::string makeType( StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { + if (info.method == FileIngestionMethod::Git && info.hash.algo != HashAlgorithm::SHA1) + throw Error("Git file ingestion must use SHA-1 hash"); + if (info.hash.algo == HashAlgorithm::SHA256 && info.method == FileIngestionMethod::Recursive) { return makeStorePath(makeType(*this, "source", info.references), info.hash, name); } else { @@ -238,7 +171,7 @@ std::pair StoreDirConfig::computeStorePath( const StorePathSet & references, PathFilter & filter) const { - auto h = hashPath(accessor, path, method.getFileIngestionMethod(), hashAlgo, filter).first; + auto h = hashPath(accessor, path, method.getFileIngestionMethod(), hashAlgo, filter); return { makeFixedOutputPathFromCA( name, @@ -264,10 +197,23 @@ StorePath Store::addToStore( PathFilter & filter, RepairFlag repair) { + FileSerialisationMethod fsm; + switch (method.getFileIngestionMethod()) { + case FileIngestionMethod::Flat: + fsm = FileSerialisationMethod::Flat; + break; + case FileIngestionMethod::Recursive: + fsm = FileSerialisationMethod::Recursive; + break; + case FileIngestionMethod::Git: + // Use NAR; Git is not a serialization method + fsm = FileSerialisationMethod::Recursive; + break; + } auto source = sinkToSource([&](Sink & sink) { - dumpPath(accessor, path, sink, method.getFileIngestionMethod(), filter); + dumpPath(accessor, path, sink, fsm, filter); }); - return addToStoreFromDump(*source, name, method, hashAlgo, references, repair); + return addToStoreFromDump(*source, name, fsm, method, hashAlgo, references, repair); } void Store::addMultipleToStore( @@ -427,9 +373,7 @@ ValidPathInfo Store::addToStoreSlow( NullFileSystemObjectSink blank; auto & parseSink = method.getFileIngestionMethod() == FileIngestionMethod::Flat ? (FileSystemObjectSink &) fileSink - : method.getFileIngestionMethod() == FileIngestionMethod::Recursive - ? (FileSystemObjectSink &) blank - : (abort(), (FileSystemObjectSink &)*(FileSystemObjectSink *)nullptr); // handled both cases + : (FileSystemObjectSink &) blank; // for recursive or git we do recursive /* The information that flows from tapped (besides being replicated in narSink), is now put in parseSink. */ @@ -441,6 +385,8 @@ ValidPathInfo Store::addToStoreSlow( auto hash = method == FileIngestionMethod::Recursive && hashAlgo == HashAlgorithm::SHA256 ? narHash + : method == FileIngestionMethod::Git + ? git::dumpHash(hashAlgo, accessor, srcPath).hash : caHashSink.finish().first; if (expectedCAHash && expectedCAHash != hash) @@ -847,7 +793,7 @@ void Store::substitutePaths(const StorePathSet & paths) if (!willSubstitute.empty()) try { std::vector subs; - for (auto & p : willSubstitute) subs.push_back(DerivedPath::Opaque{p}); + for (auto & p : willSubstitute) subs.emplace_back(DerivedPath::Opaque{p}); buildPaths(subs); } catch (Error & e) { logWarning(e.info()); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 5163070b2..5f683a211 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -466,14 +466,23 @@ public: * in `dump`, which is either a NAR serialisation (if recursive == * true) or simply the contents of a regular file (if recursive == * false). - * `dump` may be drained * - * \todo remove? + * `dump` may be drained. + * + * @param dumpMethod What serialisation format is `dump`, i.e. how + * to deserialize it. Must either match hashMethod or be + * `FileSerialisationMethod::Recursive`. + * + * @param hashMethod How content addressing? Need not match be the + * same as `dumpMethod`. + * + * @todo remove? */ virtual StorePath addToStoreFromDump( Source & dump, std::string_view name, - ContentAddressMethod method = FileIngestionMethod::Recursive, + FileSerialisationMethod dumpMethod = FileSerialisationMethod::Recursive, + ContentAddressMethod hashMethod = FileIngestionMethod::Recursive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), RepairFlag repair = NoRepair) = 0; @@ -772,7 +781,7 @@ protected: * Helper for methods that are not unsupported: this is used for * default definitions for virtual methods that are meant to be overriden. * - * \todo Using this should be a last resort. It is better to make + * @todo Using this should be a last resort. It is better to make * the method "virtual pure" and/or move it to a subclass. */ [[noreturn]] void unsupported(const std::string & op) diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index a5ac9080a..8bce8994a 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -35,7 +35,7 @@ public: static std::set uriSchemes() { return {"unix"}; } - ref getFSAccessor(bool requireValidPath) override + ref getFSAccessor(bool requireValidPath = true) override { return LocalFSStore::getFSAccessor(requireValidPath); } void narFromPath(const StorePath & path, Sink & sink) override diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 6062392cd..351ee094b 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -77,20 +77,20 @@ void SourceAccessor::dumpPath( std::string name(i.first); size_t pos = i.first.find(caseHackSuffix); if (pos != std::string::npos) { - debug("removing case hack suffix from '%s'", path + i.first); + debug("removing case hack suffix from '%s'", path / i.first); name.erase(pos); } if (!unhacked.emplace(name, i.first).second) throw Error("file name collision in between '%s' and '%s'", - (path + unhacked[name]), - (path + i.first)); + (path / unhacked[name]), + (path / i.first)); } else unhacked.emplace(i.first, i.first); for (auto & i : unhacked) - if (filter((path + i.first).abs())) { + if (filter((path / i.first).abs())) { sink << "entry" << "(" << "name" << i.first << "node"; - dump(path + i.second); + dump(path / i.second); sink << ")"; } } @@ -110,8 +110,8 @@ void SourceAccessor::dumpPath( time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter) { - PosixSourceAccessor accessor; - accessor.dumpPath(CanonPath::fromCwd(path), sink, filter); + auto [accessor, canonPath] = PosixSourceAccessor::createAtRoot(path); + accessor.dumpPath(canonPath, sink, filter); return accessor.mtime; } diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 8996cbe5b..a981ed9fb 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -544,73 +544,6 @@ nlohmann::json Args::toJSON() return res; } -static void hashFormatCompleter(AddCompletions & completions, size_t index, std::string_view prefix) -{ - for (auto & format : hashFormats) { - if (hasPrefix(format, prefix)) { - completions.add(format); - } - } -} - -Args::Flag Args::Flag::mkHashFormatFlagWithDefault(std::string &&longName, HashFormat * hf) { - assert(*hf == nix::HashFormat::SRI); - return Flag{ - .longName = std::move(longName), - .description = "Hash format (`base16`, `nix32`, `base64`, `sri`). Default: `sri`.", - .labels = {"hash-format"}, - .handler = {[hf](std::string s) { - *hf = parseHashFormat(s); - }}, - .completer = hashFormatCompleter, - }; -} - -Args::Flag Args::Flag::mkHashFormatOptFlag(std::string && longName, std::optional * ohf) { - return Flag{ - .longName = std::move(longName), - .description = "Hash format (`base16`, `nix32`, `base64`, `sri`).", - .labels = {"hash-format"}, - .handler = {[ohf](std::string s) { - *ohf = std::optional{parseHashFormat(s)}; - }}, - .completer = hashFormatCompleter, - }; -} - -static void hashAlgoCompleter(AddCompletions & completions, size_t index, std::string_view prefix) -{ - for (auto & algo : hashAlgorithms) - if (hasPrefix(algo, prefix)) - completions.add(algo); -} - -Args::Flag Args::Flag::mkHashAlgoFlag(std::string && longName, HashAlgorithm * ha) -{ - return Flag{ - .longName = std::move(longName), - .description = "Hash algorithm (`md5`, `sha1`, `sha256`, or `sha512`).", - .labels = {"hash-algo"}, - .handler = {[ha](std::string s) { - *ha = parseHashAlgo(s); - }}, - .completer = hashAlgoCompleter, - }; -} - -Args::Flag Args::Flag::mkHashAlgoOptFlag(std::string && longName, std::optional * oha) -{ - return Flag{ - .longName = std::move(longName), - .description = "Hash algorithm (`md5`, `sha1`, `sha256`, or `sha512`). Can be omitted for SRI hashes.", - .labels = {"hash-algo"}, - .handler = {[oha](std::string s) { - *oha = std::optional{parseHashAlgo(s)}; - }}, - .completer = hashAlgoCompleter, - }; -} - static void _completePath(AddCompletions & completions, std::string_view prefix, bool onlyDirs) { completions.setType(Completions::Type::Filenames); diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 6c9c48065..4b2e1d960 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -155,6 +155,8 @@ protected: */ using CompleterClosure = std::function; +public: + /** * Description of flags / options * @@ -175,19 +177,10 @@ protected: CompleterClosure completer; std::optional experimentalFeature; - - static Flag mkHashAlgoFlag(std::string && longName, HashAlgorithm * ha); - static Flag mkHashAlgoFlag(HashAlgorithm * ha) { - return mkHashAlgoFlag("hash-algo", ha); - } - static Flag mkHashAlgoOptFlag(std::string && longName, std::optional * oha); - static Flag mkHashAlgoOptFlag(std::optional * oha) { - return mkHashAlgoOptFlag("hash-algo", oha); - } - static Flag mkHashFormatFlagWithDefault(std::string && longName, HashFormat * hf); - static Flag mkHashFormatOptFlag(std::string && longName, std::optional * ohf); }; +protected: + /** * Index of all registered "long" flag descriptions (flags like * `--long`). @@ -206,6 +199,8 @@ protected: */ virtual bool processFlag(Strings::iterator & pos, Strings::iterator end); +public: + /** * Description of positional arguments * @@ -220,6 +215,8 @@ protected: CompleterClosure completer; }; +protected: + /** * Queue of expected positional argument forms. * diff --git a/src/libutil/canon-path.cc b/src/libutil/canon-path.cc index 0a0f96a05..27f048697 100644 --- a/src/libutil/canon-path.cc +++ b/src/libutil/canon-path.cc @@ -1,16 +1,25 @@ #include "canon-path.hh" -#include "file-system.hh" +#include "util.hh" +#include "file-path-impl.hh" namespace nix { CanonPath CanonPath::root = CanonPath("/"); +static std::string absPathPure(std::string_view path) +{ + return canonPathInner(path, [](auto &, auto &){}); +} + CanonPath::CanonPath(std::string_view raw) - : path(absPath(raw, "/")) + : path(absPathPure(concatStrings("/", raw))) { } CanonPath::CanonPath(std::string_view raw, const CanonPath & root) - : path(absPath(raw, root.abs())) + : path(absPathPure( + raw.size() > 0 && raw[0] == '/' + ? raw + : concatStrings(root.abs(), "/", raw))) { } CanonPath::CanonPath(const std::vector & elems) @@ -20,11 +29,6 @@ CanonPath::CanonPath(const std::vector & elems) push(s); } -CanonPath CanonPath::fromCwd(std::string_view path) -{ - return CanonPath(unchecked_t(), absPath(path)); -} - std::optional CanonPath::parent() const { if (isRoot()) return std::nullopt; @@ -63,7 +67,7 @@ void CanonPath::extend(const CanonPath & x) path += x.abs(); } -CanonPath CanonPath::operator + (const CanonPath & x) const +CanonPath CanonPath::operator / (const CanonPath & x) const { auto res = *this; res.extend(x); @@ -78,7 +82,7 @@ void CanonPath::push(std::string_view c) path += c; } -CanonPath CanonPath::operator + (std::string_view c) const +CanonPath CanonPath::operator / (std::string_view c) const { auto res = *this; res.push(c); diff --git a/src/libutil/canon-path.hh b/src/libutil/canon-path.hh index 997c8c731..8f5a1c279 100644 --- a/src/libutil/canon-path.hh +++ b/src/libutil/canon-path.hh @@ -21,9 +21,21 @@ namespace nix { * * - There are no components equal to '.' or '..'. * - * Note that the path does not need to correspond to an actually - * existing path, and there is no guarantee that symlinks are - * resolved. + * `CanonPath` are "virtual" Nix paths for abstract file system objects; + * they are always Unix-style paths, regardless of what OS Nix is + * running on. The `/` root doesn't denote the ambient host file system + * root, but some virtual FS root. + * + * @note It might be useful to compare `openat(some_fd, "foo/bar")` on + * Unix. `"foo/bar"` is a relative path because an absolute path would + * "override" the `some_fd` directory file descriptor and escape to the + * "system root". Conversely, Nix's abstract file operations *never* escape the + * designated virtual file system (i.e. `SourceAccessor` or + * `ParseSink`), so `CanonPath` does not need an absolute/relative + * distinction. + * + * @note The path does not need to correspond to an actually existing + * path, and the path may or may not have unresolved symlinks. */ class CanonPath { @@ -52,8 +64,6 @@ public: */ CanonPath(const std::vector & elems); - static CanonPath fromCwd(std::string_view path = "."); - static CanonPath root; /** @@ -190,14 +200,14 @@ public: /** * Concatenate two paths. */ - CanonPath operator + (const CanonPath & x) const; + CanonPath operator / (const CanonPath & x) const; /** * Add a path component to this one. It must not contain any slashes. */ void push(std::string_view c); - CanonPath operator + (std::string_view c) const; + CanonPath operator / (std::string_view c) const; /** * Check whether access to this path is allowed, which is the case diff --git a/src/libutil/config-impl.hh b/src/libutil/config-impl.hh index 9f69e8444..1da0cb638 100644 --- a/src/libutil/config-impl.hh +++ b/src/libutil/config-impl.hh @@ -4,7 +4,7 @@ * * Template implementations (as opposed to mere declarations). * - * This file is an exmample of the "impl.hh" pattern. See the + * This file is an example of the "impl.hh" pattern. See the * contributing guide. * * One only needs to include this when one is declaring a diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 37f5b50c7..617c2ec89 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -84,7 +84,9 @@ void AbstractConfig::reapplyUnknownSettings() void Config::getSettings(std::map & res, bool overriddenOnly) { for (const auto & opt : _settings) - if (!opt.second.isAlias && (!overriddenOnly || opt.second.setting->overridden)) + if (!opt.second.isAlias + && (!overriddenOnly || opt.second.setting->overridden) + && experimentalFeatureSettings.isEnabled(opt.second.setting->experimentalFeature)) res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description}); } diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc index 01f64f211..47aa137d8 100644 --- a/src/libutil/current-process.cc +++ b/src/libutil/current-process.cc @@ -63,7 +63,7 @@ void setStackSize(rlim_t stackSize) if (setrlimit(RLIMIT_STACK, &limit) != 0) { logger->log( lvlError, - hintfmt( + HintFmt( "Failed to increase stack size from %1% to %2% (maximum allowed stack size: %3%): %4%", savedStackSize, stackSize, diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 1f0cb08c9..d1e864a1a 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -11,9 +11,9 @@ namespace nix { -void BaseError::addTrace(std::shared_ptr && e, hintformat hint, bool frame) +void BaseError::addTrace(std::shared_ptr && e, HintFmt hint) { - err.traces.push_front(Trace { .pos = std::move(e), .hint = hint, .frame = frame }); + err.traces.push_front(Trace { .pos = std::move(e), .hint = hint }); } void throwExceptionSelfCheck(){ @@ -37,7 +37,7 @@ const std::string & BaseError::calcWhat() const std::optional ErrorInfo::programName = std::nullopt; -std::ostream & operator <<(std::ostream & os, const hintformat & hf) +std::ostream & operator <<(std::ostream & os, const HintFmt & hf) { return os << hf.str(); } @@ -61,8 +61,7 @@ inline bool operator<(const Trace& lhs, const Trace& rhs) // This formats a freshly formatted hint string and then throws it away, which // shouldn't be much of a problem because it only runs when pos is equal, and this function is // used for trace printing, which is infrequent. - return std::forward_as_tuple(lhs.hint.str(), lhs.frame) - < std::forward_as_tuple(rhs.hint.str(), rhs.frame); + return lhs.hint.str() < rhs.hint.str(); } inline bool operator> (const Trace& lhs, const Trace& rhs) { return rhs < lhs; } inline bool operator<=(const Trace& lhs, const Trace& rhs) { return !(lhs > rhs); } @@ -335,7 +334,7 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s * try { * e->eval(*this, env, v); * if (v.type() != nAttrs) - * throwTypeError("expected a set but found %1%", v); + * error("expected a set but found %1%", v); * } catch (Error & e) { * e.addTrace(pos, errorCtx); * throw; @@ -349,7 +348,7 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s * e->eval(*this, env, v); * try { * if (v.type() != nAttrs) - * throwTypeError("expected a set but found %1%", v); + * error("expected a set but found %1%", v); * } catch (Error & e) { * e.addTrace(pos, errorCtx); * throw; @@ -373,7 +372,6 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s // prepended to each element of the trace auto ellipsisIndent = " "; - bool frameOnly = false; if (!einfo.traces.empty()) { // Stack traces seen since we last printed a chunk of `duplicate frames // omitted`. @@ -384,7 +382,6 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s for (const auto & trace : einfo.traces) { if (trace.hint.str().empty()) continue; - if (frameOnly && !trace.frame) continue; if (!showTrace && count > 3) { oss << "\n" << ANSI_WARNING "(stack trace truncated; use '--show-trace' to show the full trace)" ANSI_NORMAL << "\n"; @@ -400,7 +397,6 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s printSkippedTracesMaybe(oss, ellipsisIndent, count, skippedTraces, tracesSeen); count++; - frameOnly = trace.frame; printTrace(oss, ellipsisIndent, count, trace); } @@ -411,7 +407,7 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s oss << einfo.msg << "\n"; - printPosMaybe(oss, "", einfo.errPos); + printPosMaybe(oss, "", einfo.pos); auto suggestions = einfo.suggestions.trim(); if (!suggestions.suggestions.empty()) { diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 764fac1ce..89f5ad021 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -31,15 +31,6 @@ #include #include -/* Before 4.7, gcc's std::exception uses empty throw() specifiers for - * its (virtual) destructor and what() in c++11 mode, in violation of spec - */ -#ifdef __GNUC__ -#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 7) -#define EXCEPTION_NEEDS_THROW_SPEC -#endif -#endif - namespace nix { @@ -72,8 +63,7 @@ void printCodeLines(std::ostream & out, struct Trace { std::shared_ptr pos; - hintformat hint; - bool frame; + HintFmt hint; }; inline bool operator<(const Trace& lhs, const Trace& rhs); @@ -83,10 +73,15 @@ inline bool operator>=(const Trace& lhs, const Trace& rhs); struct ErrorInfo { Verbosity level; - hintformat msg; - std::shared_ptr errPos; + HintFmt msg; + std::shared_ptr pos; std::list traces; + /** + * Exit status. + */ + unsigned int status = 1; + Suggestions suggestions; static std::optional programName; @@ -103,31 +98,34 @@ class BaseError : public std::exception protected: mutable ErrorInfo err; + /** + * Cached formatted contents of `err.msg`. + */ mutable std::optional what_; + /** + * Format `err.msg` and set `what_` to the resulting value. + */ const std::string & calcWhat() const; public: - unsigned int status = 1; // exit status - BaseError(const BaseError &) = default; template BaseError(unsigned int status, const Args & ... args) - : err { .level = lvlError, .msg = hintfmt(args...) } - , status(status) + : err { .level = lvlError, .msg = HintFmt(args...), .status = status } { } template explicit BaseError(const std::string & fs, const Args & ... args) - : err { .level = lvlError, .msg = hintfmt(fs, args...) } + : err { .level = lvlError, .msg = HintFmt(fs, args...) } { } template BaseError(const Suggestions & sug, const Args & ... args) - : err { .level = lvlError, .msg = hintfmt(args...), .suggestions = sug } + : err { .level = lvlError, .msg = HintFmt(args...), .suggestions = sug } { } - BaseError(hintformat hint) + BaseError(HintFmt hint) : err { .level = lvlError, .msg = hint } { } @@ -139,16 +137,19 @@ public: : err(e) { } -#ifdef EXCEPTION_NEEDS_THROW_SPEC - ~BaseError() throw () { }; - const char * what() const throw () { return calcWhat().c_str(); } -#else const char * what() const noexcept override { return calcWhat().c_str(); } -#endif - const std::string & msg() const { return calcWhat(); } const ErrorInfo & info() const { calcWhat(); return err; } + void withExitStatus(unsigned int status) + { + err.status = status; + } + + void atPos(std::shared_ptr pos) { + err.pos = pos; + } + void pushTrace(Trace trace) { err.traces.push_front(trace); @@ -157,10 +158,10 @@ public: template void addTrace(std::shared_ptr && e, std::string_view fs, const Args & ... args) { - addTrace(std::move(e), hintfmt(std::string(fs), args...)); + addTrace(std::move(e), HintFmt(std::string(fs), args...)); } - void addTrace(std::shared_ptr && e, hintformat hint, bool frame = false); + void addTrace(std::shared_ptr && e, HintFmt hint); bool hasTrace() const { return !err.traces.empty(); } @@ -212,8 +213,8 @@ public: SysError(int errNo, const Args & ... args) : SystemError(""), errNo(errNo) { - auto hf = hintfmt(args...); - err.msg = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo)); + auto hf = HintFmt(args...); + err.msg = HintFmt("%1%: %2%", Uncolored(hf.str()), strerror(errNo)); } /** diff --git a/src/libutil/exit.cc b/src/libutil/exit.cc new file mode 100644 index 000000000..73cd8b04e --- /dev/null +++ b/src/libutil/exit.cc @@ -0,0 +1,7 @@ +#include "exit.hh" + +namespace nix { + +Exit::~Exit() {} + +} diff --git a/src/libutil/exit.hh b/src/libutil/exit.hh new file mode 100644 index 000000000..55f33e62f --- /dev/null +++ b/src/libutil/exit.hh @@ -0,0 +1,19 @@ +#pragma once + +#include + +namespace nix { + +/** + * Exit the program with a given exit code. + */ +class Exit : public std::exception +{ +public: + int status; + Exit() : status(0) { } + explicit Exit(int status) : status(status) { } + virtual ~Exit(); +}; + +} diff --git a/src/libutil/file-content-address.cc b/src/libutil/file-content-address.cc index 9917986f6..471bda6a0 100644 --- a/src/libutil/file-content-address.cc +++ b/src/libutil/file-content-address.cc @@ -1,19 +1,83 @@ #include "file-content-address.hh" #include "archive.hh" +#include "git.hh" namespace nix { +static std::optional parseFileSerialisationMethodOpt(std::string_view input) +{ + if (input == "flat") { + return FileSerialisationMethod::Flat; + } else if (input == "nar") { + return FileSerialisationMethod::Recursive; + } else { + return std::nullopt; + } +} + +FileSerialisationMethod parseFileSerialisationMethod(std::string_view input) +{ + auto ret = parseFileSerialisationMethodOpt(input); + if (ret) + return *ret; + else + throw UsageError("Unknown file serialiation method '%s', expect `flat` or `nar`"); +} + + +FileIngestionMethod parseFileIngestionMethod(std::string_view input) +{ + if (input == "git") { + return FileIngestionMethod::Git; + } else { + auto ret = parseFileSerialisationMethodOpt(input); + if (ret) + return static_cast(*ret); + else + throw UsageError("Unknown file ingestion method '%s', expect `flat`, `nar`, or `git`"); + } +} + + +std::string_view renderFileSerialisationMethod(FileSerialisationMethod method) +{ + switch (method) { + case FileSerialisationMethod::Flat: + return "flat"; + case FileSerialisationMethod::Recursive: + return "nar"; + default: + assert(false); + } +} + + +std::string_view renderFileIngestionMethod(FileIngestionMethod method) +{ + switch (method) { + case FileIngestionMethod::Flat: + case FileIngestionMethod::Recursive: + return renderFileSerialisationMethod( + static_cast(method)); + case FileIngestionMethod::Git: + return "git"; + default: + abort(); + } +} + + void dumpPath( SourceAccessor & accessor, const CanonPath & path, Sink & sink, - FileIngestionMethod method, + FileSerialisationMethod method, PathFilter & filter) { switch (method) { - case FileIngestionMethod::Flat: + case FileSerialisationMethod::Flat: accessor.readFile(path, sink); break; - case FileIngestionMethod::Recursive: + case FileSerialisationMethod::Recursive: accessor.dumpPath(path, sink, filter); break; } @@ -23,13 +87,13 @@ void dumpPath( void restorePath( const Path & path, Source & source, - FileIngestionMethod method) + FileSerialisationMethod method) { switch (method) { - case FileIngestionMethod::Flat: + case FileSerialisationMethod::Flat: writeFile(path, source); break; - case FileIngestionMethod::Recursive: + case FileSerialisationMethod::Recursive: restorePath(path, source); break; } @@ -38,12 +102,28 @@ void restorePath( HashResult hashPath( SourceAccessor & accessor, const CanonPath & path, - FileIngestionMethod method, HashAlgorithm ht, + FileSerialisationMethod method, HashAlgorithm ha, PathFilter & filter) { - HashSink sink { ht }; + HashSink sink { ha }; dumpPath(accessor, path, sink, method, filter); return sink.finish(); } + +Hash hashPath( + SourceAccessor & accessor, const CanonPath & path, + FileIngestionMethod method, HashAlgorithm ht, + PathFilter & filter) +{ + switch (method) { + case FileIngestionMethod::Flat: + case FileIngestionMethod::Recursive: + return hashPath(accessor, path, (FileSerialisationMethod) method, ht, filter).first; + case FileIngestionMethod::Git: + return git::dumpHash(ht, accessor, path, filter).hash; + } + +} + } diff --git a/src/libutil/file-content-address.hh b/src/libutil/file-content-address.hh index 7f7544e41..b361ab243 100644 --- a/src/libutil/file-content-address.hh +++ b/src/libutil/file-content-address.hh @@ -8,49 +8,138 @@ namespace nix { /** - * An enumeration of the main ways we can serialize file system + * An enumeration of the ways we can serialize file system * objects. */ -enum struct FileIngestionMethod : uint8_t { +enum struct FileSerialisationMethod : uint8_t { /** - * Flat-file hashing. Directly ingest the contents of a single file + * Flat-file. The contents of a single file exactly. */ - Flat = 0, + Flat, + /** - * Recursive (or NAR) hashing. Serializes the file-system object in - * Nix Archive format and ingest that. + * Nix Archive. Serializes the file-system object in + * Nix Archive format. */ - Recursive = 1, + Recursive, }; +/** + * Parse a `FileSerialisationMethod` by name. Choice of: + * + * - `flat`: `FileSerialisationMethod::Flat` + * - `nar`: `FileSerialisationMethod::Recursive` + * + * Opposite of `renderFileSerialisationMethod`. + */ +FileSerialisationMethod parseFileSerialisationMethod(std::string_view input); + +/** + * Render a `FileSerialisationMethod` by name. + * + * Opposite of `parseFileSerialisationMethod`. + */ +std::string_view renderFileSerialisationMethod(FileSerialisationMethod method); + /** * Dump a serialization of the given file system object. */ void dumpPath( SourceAccessor & accessor, const CanonPath & path, Sink & sink, - FileIngestionMethod method, + FileSerialisationMethod method, PathFilter & filter = defaultPathFilter); /** - * Restore a serialization of the given file system object. + * Restore a serialisation of the given file system object. * * @TODO use an arbitrary `FileSystemObjectSink`. */ void restorePath( const Path & path, Source & source, - FileIngestionMethod method); + FileSerialisationMethod method); + /** * Compute the hash of the given file system object according to the * given method. * - * The hash is defined as (essentially) hashString(ht, dumpPath(path)). + * the hash is defined as (in pseudocode): + * + * ``` + * hashString(ha, dumpPath(...)) + * ``` */ HashResult hashPath( SourceAccessor & accessor, const CanonPath & path, - FileIngestionMethod method, HashAlgorithm ht, + FileSerialisationMethod method, HashAlgorithm ha, + PathFilter & filter = defaultPathFilter); + +/** + * An enumeration of the ways we can ingest file system + * objects, producing a hash or digest. + */ +enum struct FileIngestionMethod : uint8_t { + /** + * Hash `FileSerialisationMethod::Flat` serialisation. + */ + Flat, + + /** + * Hash `FileSerialisationMethod::Git` serialisation. + */ + Recursive, + + /** + * Git hashing. In particular files are hashed as git "blobs", and + * directories are hashed as git "trees". + * + * Unlike `Flat` and `Recursive`, this is not a hash of a single + * serialisation but a [Merkle + * DAG](https://en.wikipedia.org/wiki/Merkle_tree) of multiple + * rounds of serialisation and hashing. + * + * @note Git's data model is slightly different, in that a plain + * file doesn't have an executable bit, directory entries do + * instead. We decide treat a bare file as non-executable by fiat, + * as we do with `FileIngestionMethod::Flat` which also lacks this + * information. Thus, Git can encode some but all of Nix's "File + * System Objects", and this sort of hashing is likewise partial. + */ + Git, +}; + +/** + * Parse a `FileIngestionMethod` by name. Choice of: + * + * - `flat`: `FileIngestionMethod::Flat` + * - `nar`: `FileIngestionMethod::Recursive` + * - `git`: `FileIngestionMethod::Git` + * + * Opposite of `renderFileIngestionMethod`. + */ +FileIngestionMethod parseFileIngestionMethod(std::string_view input); + +/** + * Render a `FileIngestionMethod` by name. + * + * Opposite of `parseFileIngestionMethod`. + */ +std::string_view renderFileIngestionMethod(FileIngestionMethod method); + +/** + * Compute the hash of the given file system object according to the + * given method. + * + * Unlike the other `hashPath`, this works on an arbitrary + * `FileIngestionMethod` instead of `FileSerialisationMethod`, but + * doesn't return the size as this is this is not a both simple and + * useful defined for a merkle format. + */ +Hash hashPath( + SourceAccessor & accessor, const CanonPath & path, + FileIngestionMethod method, HashAlgorithm ha, PathFilter & filter = defaultPathFilter); } diff --git a/src/libutil/file-path-impl.hh b/src/libutil/file-path-impl.hh new file mode 100644 index 000000000..4c90150fd --- /dev/null +++ b/src/libutil/file-path-impl.hh @@ -0,0 +1,176 @@ +#pragma once +/** + * @file + * + * Pure (no IO) infrastructure just for defining other path types; + * should not be used directly outside of utilities. + */ +#include +#include + +namespace nix { + +/** + * Unix-style path primives. + * + * Nix'result own "logical" paths are always Unix-style. So this is always + * used for that, and additionally used for native paths on Unix. + */ +struct UnixPathTrait +{ + using CharT = char; + + using String = std::string; + + using StringView = std::string_view; + + constexpr static char preferredSep = '/'; + + static inline bool isPathSep(char c) + { + return c == '/'; + } + + static inline size_t findPathSep(StringView path, size_t from = 0) + { + return path.find('/', from); + } + + static inline size_t rfindPathSep(StringView path, size_t from = StringView::npos) + { + return path.rfind('/', from); + } +}; + + +/** + * Windows-style path primitives. + * + * The character type is a parameter because while windows paths rightly + * work over UTF-16 (*) using `wchar_t`, at the current time we are + * often manipulating them converted to UTF-8 (*) using `char`. + * + * (Actually neither are guaranteed to be valid unicode; both are + * arbitrary non-0 8- or 16-bit bytes. But for charcters with specifical + * meaning like '/', '\\', ':', etc., we refer to an encoding scheme, + * and also for sake of UIs that display paths a text.) + */ +template +struct WindowsPathTrait +{ + using CharT = CharT0; + + using String = std::basic_string; + + using StringView = std::basic_string_view; + + constexpr static CharT preferredSep = '\\'; + + static inline bool isPathSep(CharT c) + { + return c == '/' || c == preferredSep; + } + + static size_t findPathSep(StringView path, size_t from = 0) + { + size_t p1 = path.find('/', from); + size_t p2 = path.find(preferredSep, from); + return p1 == String::npos ? p2 : + p2 == String::npos ? p1 : + std::min(p1, p2); + } + + static size_t rfindPathSep(StringView path, size_t from = String::npos) + { + size_t p1 = path.rfind('/', from); + size_t p2 = path.rfind(preferredSep, from); + return p1 == String::npos ? p2 : + p2 == String::npos ? p1 : + std::max(p1, p2); + } +}; + + +/** + * @todo Revisit choice of `char` or `wchar_t` for `WindowsPathTrait` + * argument. + */ +using NativePathTrait = +#ifdef _WIN32 + WindowsPathTrait +#else + UnixPathTrait +#endif + ; + + +/** + * Core pure path canonicalization algorithm. + * + * @param hookComponent + * A callback which is passed two arguments, + * references to + * + * 1. the result so far + * + * 2. the remaining path to resolve + * + * This is a chance to modify those two paths in arbitrary way, e.g. if + * "result" points to a symlink. + */ +template +typename PathDict::String canonPathInner( + typename PathDict::StringView remaining, + auto && hookComponent) +{ + assert(remaining != ""); + + typename PathDict::String result; + result.reserve(256); + + while (true) { + + /* Skip slashes. */ + while (!remaining.empty() && PathDict::isPathSep(remaining[0])) + remaining.remove_prefix(1); + + if (remaining.empty()) break; + + auto nextComp = ({ + auto nextPathSep = PathDict::findPathSep(remaining); + nextPathSep == remaining.npos ? remaining : remaining.substr(0, nextPathSep); + }); + + /* Ignore `.'. */ + if (nextComp == ".") + remaining.remove_prefix(1); + + /* If `..', delete the last component. */ + else if (nextComp == "..") + { + if (!result.empty()) result.erase(PathDict::rfindPathSep(result)); + remaining.remove_prefix(2); + } + + /* Normal component; copy it. */ + else { + result += PathDict::preferredSep; + if (const auto slash = PathDict::findPathSep(remaining); slash == result.npos) { + result += remaining; + remaining = {}; + } else { + result += remaining.substr(0, slash); + remaining = remaining.substr(slash); + } + + hookComponent(result, remaining); + } + } + + if (result.empty()) + result = typename PathDict::String { PathDict::preferredSep }; + + return result; +} + +} diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index cf8a6d967..b0a3f0797 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -1,5 +1,6 @@ #include "environment-variables.hh" #include "file-system.hh" +#include "file-path-impl.hh" #include "signals.hh" #include "finally.hh" #include "serialise.hh" @@ -21,11 +22,22 @@ namespace fs = std::filesystem; namespace nix { +/** + * Treat the string as possibly an absolute path, by inspecting the + * start of it. Return whether it was probably intended to be + * absolute. + */ +static bool isAbsolute(PathView path) +{ + return fs::path { path }.is_absolute(); +} + + Path absPath(PathView path, std::optional dir, bool resolveSymlinks) { std::string scratch; - if (path[0] != '/') { + if (!isAbsolute(path)) { // In this case we need to call `canonPath` on a newly-created // string. We set `scratch` to that string first, and then set // `path` to `scratch`. This ensures the newly-created string @@ -58,69 +70,46 @@ Path canonPath(PathView path, bool resolveSymlinks) { assert(path != ""); - std::string s; - s.reserve(256); - - if (path[0] != '/') + if (!isAbsolute(path)) throw Error("not an absolute path: '%1%'", path); + // For Windows + auto rootName = fs::path { path }.root_name(); + + /* This just exists because we cannot set the target of `remaining` + (the callback parameter) directly to a newly-constructed string, + since it is `std::string_view`. */ std::string temp; /* Count the number of times we follow a symlink and stop at some arbitrary (but high) limit to prevent infinite loops. */ unsigned int followCount = 0, maxFollow = 1024; - while (1) { - - /* Skip slashes. */ - while (!path.empty() && path[0] == '/') path.remove_prefix(1); - if (path.empty()) break; - - /* Ignore `.'. */ - if (path == "." || path.substr(0, 2) == "./") - path.remove_prefix(1); - - /* If `..', delete the last component. */ - else if (path == ".." || path.substr(0, 3) == "../") - { - if (!s.empty()) s.erase(s.rfind('/')); - path.remove_prefix(2); - } - - /* Normal component; copy it. */ - else { - s += '/'; - if (const auto slash = path.find('/'); slash == path.npos) { - s += path; - path = {}; - } else { - s += path.substr(0, slash); - path = path.substr(slash); - } - - /* If s points to a symlink, resolve it and continue from there */ - if (resolveSymlinks && isLink(s)) { + auto ret = canonPathInner( + path, + [&followCount, &temp, maxFollow, resolveSymlinks] + (std::string & result, std::string_view & remaining) { + if (resolveSymlinks && isLink(result)) { if (++followCount >= maxFollow) - throw Error("infinite symlink recursion in path '%1%'", path); - temp = concatStrings(readLink(s), path); - path = temp; - if (!temp.empty() && temp[0] == '/') { - s.clear(); /* restart for symlinks pointing to absolute path */ + throw Error("infinite symlink recursion in path '%0%'", remaining); + remaining = (temp = concatStrings(readLink(result), remaining)); + if (isAbsolute(remaining)) { + /* restart for symlinks pointing to absolute path */ + result.clear(); } else { - s = dirOf(s); - if (s == "/") { // we don’t want trailing slashes here, which dirOf only produces if s = / - s.clear(); + result = dirOf(result); + if (result == "/") { + /* we don’t want trailing slashes here, which `dirOf` + only produces if `result = /` */ + result.clear(); } } } - } - } + }); - if (s.empty()) { - s = "/"; - } - - return s; + if (!rootName.empty()) + ret = rootName.string() + std::move(ret); + return ret; } diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index ac72e47fb..e996f4ba2 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -8,37 +8,64 @@ namespace nix { - +namespace { /** - * Inherit some names from other namespaces for convenience. - */ -using boost::format; - - -/** - * A variadic template that does nothing. Useful to call a function - * for all variadic arguments but ignoring the result. - */ -struct nop { template nop(T...) {} }; - - -/** - * A helper for formatting strings. ‘fmt(format, a_0, ..., a_n)’ is - * equivalent to ‘boost::format(format) % a_0 % ... % - * ... a_n’. However, ‘fmt(s)’ is equivalent to ‘s’ (so no %-expansion - * takes place). + * A helper for writing `boost::format` expressions. + * + * These are equivalent: + * + * ``` + * formatHelper(formatter, a_0, ..., a_n) + * formatter % a_0 % ... % a_n + * ``` + * + * With a single argument, `formatHelper(s)` is a no-op. */ template inline void formatHelper(F & f) -{ -} +{ } template inline void formatHelper(F & f, const T & x, const Args & ... args) { + // Interpolate one argument and then recurse. formatHelper(f % x, args...); } +/** + * Set the correct exceptions for `fmt`. + */ +void setExceptions(boost::format & fmt) +{ + fmt.exceptions( + boost::io::all_error_bits ^ + boost::io::too_many_args_bit ^ + boost::io::too_few_args_bit); +} +} + +/** + * A helper for writing a `boost::format` expression to a string. + * + * These are (roughly) equivalent: + * + * ``` + * fmt(formatString, a_0, ..., a_n) + * (boost::format(formatString) % a_0 % ... % a_n).str() + * ``` + * + * However, when called with a single argument, the string is returned + * unchanged. + * + * If you write code like this: + * + * ``` + * std::cout << boost::format(stringFromUserInput) << std::endl; + * ``` + * + * And `stringFromUserInput` contains formatting placeholders like `%s`, then + * the code will crash at runtime. `fmt` helps you avoid this pitfall. + */ inline std::string fmt(const std::string & s) { return s; @@ -58,68 +85,96 @@ template inline std::string fmt(const std::string & fs, const Args & ... args) { boost::format f(fs); - f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); + setExceptions(f); formatHelper(f, args...); return f.str(); } -// ----------------------------------------------------------------------------- -// format function for hints in errors. same as fmt, except templated values -// are always in yellow. - +/** + * Values wrapped in this struct are printed in magenta. + * + * By default, arguments to `HintFmt` are printed in magenta. To avoid this, + * either wrap the argument in `Uncolored` or add a specialization of + * `HintFmt::operator%`. + */ template -struct yellowtxt +struct Magenta { - yellowtxt(const T &s) : value(s) {} + Magenta(const T &s) : value(s) {} const T & value; }; template -std::ostream & operator<<(std::ostream & out, const yellowtxt & y) +std::ostream & operator<<(std::ostream & out, const Magenta & y) { return out << ANSI_WARNING << y.value << ANSI_NORMAL; } +/** + * Values wrapped in this class are printed without coloring. + * + * By default, arguments to `HintFmt` are printed in magenta (see `Magenta`). + */ template -struct normaltxt +struct Uncolored { - normaltxt(const T & s) : value(s) {} + Uncolored(const T & s) : value(s) {} const T & value; }; template -std::ostream & operator<<(std::ostream & out, const normaltxt & y) +std::ostream & operator<<(std::ostream & out, const Uncolored & y) { return out << ANSI_NORMAL << y.value; } -class hintformat +/** + * A wrapper around `boost::format` which colors interpolated arguments in + * magenta by default. + */ +class HintFmt { -public: - hintformat(const std::string & format) : fmt(format) - { - fmt.exceptions(boost::io::all_error_bits ^ - boost::io::too_many_args_bit ^ - boost::io::too_few_args_bit); - } +private: + boost::format fmt; - hintformat(const hintformat & hf) +public: + /** + * Format the given string literally, without interpolating format + * placeholders. + */ + HintFmt(const std::string & literal) + : HintFmt("%s", Uncolored(literal)) + { } + + /** + * Interpolate the given arguments into the format string. + */ + template + HintFmt(const std::string & format, const Args & ... args) + : HintFmt(boost::format(format), args...) + { } + + HintFmt(const HintFmt & hf) : fmt(hf.fmt) { } - hintformat(format && fmt) + template + HintFmt(boost::format && fmt, const Args & ... args) : fmt(std::move(fmt)) - { } + { + setExceptions(fmt); + formatHelper(*this, args...); + } template - hintformat & operator%(const T & value) + HintFmt & operator%(const T & value) { - fmt % yellowtxt(value); + fmt % Magenta(value); return *this; } template - hintformat & operator%(const normaltxt & value) + HintFmt & operator%(const Uncolored & value) { fmt % value.value; return *this; @@ -129,25 +184,8 @@ public: { return fmt.str(); } - -private: - format fmt; }; -std::ostream & operator<<(std::ostream & os, const hintformat & hf); - -template -inline hintformat hintfmt(const std::string & fs, const Args & ... args) -{ - hintformat f(fs); - formatHelper(f, args...); - return f; -} - -inline hintformat hintfmt(const std::string & plain_string) -{ - // we won't be receiving any args in this case, so just print the original string - return hintfmt("%s", normaltxt(plain_string)); -} +std::ostream & operator<<(std::ostream & os, const HintFmt & hf); } diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc index b6f8db592..35ce0ac36 100644 --- a/src/libutil/fs-sink.cc +++ b/src/libutil/fs-sink.cc @@ -15,6 +15,7 @@ void copyRecursive( case SourceAccessor::tSymlink: { sink.createSymlink(to, accessor.readLink(from)); + break; } case SourceAccessor::tRegular: @@ -34,10 +35,11 @@ void copyRecursive( sink.createDirectory(to); for (auto & [name, _] : accessor.readDirectory(from)) { copyRecursive( - accessor, from + name, + accessor, from / name, sink, to + "/" + name); break; } + break; } case SourceAccessor::tMisc: diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh index 4dfb5b329..ae577819a 100644 --- a/src/libutil/fs-sink.hh +++ b/src/libutil/fs-sink.hh @@ -26,6 +26,8 @@ struct CreateRegularFileSink : Sink struct FileSystemObjectSink { + virtual ~FileSystemObjectSink() = default; + virtual void createDirectory(const Path & path) = 0; /** diff --git a/src/libutil/git.cc b/src/libutil/git.cc index 3b8c3ebac..5733531fa 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -259,7 +259,7 @@ Mode dump( { Tree entries; for (auto & [name, _] : accessor.readDirectory(path)) { - auto child = path + name; + auto child = path / name; if (!filter(child.abs())) continue; auto entry = hook(child); diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index d067da969..d4c9d6533 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -274,7 +274,7 @@ Hash newHashAllowEmpty(std::string_view hashStr, std::optional ha { if (hashStr.empty()) { if (!ha) - throw BadHash("empty hash requires explicit hash type"); + throw BadHash("empty hash requires explicit hash algorithm"); Hash h(*ha); warn("found empty hash, assuming '%s'", h.to_string(HashFormat::SRI, true)); return h; diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index f7e8eb265..e14aae43c 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -58,7 +58,7 @@ struct Hash * Parse the hash from a string representation in the format * "[:]" or "-" (a * Subresource Integrity hash expression). If the 'type' argument - * is not present, then the hash type must be specified in the + * is not present, then the hash algorithm must be specified in the * string. */ static Hash parseAny(std::string_view s, std::optional optAlgo); @@ -200,7 +200,7 @@ std::optional parseHashFormatOpt(std::string_view hashFormatName); std::string_view printHashFormat(HashFormat hashFormat); /** - * Parse a string representing a hash type. + * Parse a string representing a hash algorithm. */ HashAlgorithm parseHashAlgo(std::string_view s); diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index d68ddacc0..89fbd194a 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -199,7 +199,7 @@ struct JSONLogger : Logger { json["level"] = ei.level; json["msg"] = oss.str(); json["raw_msg"] = ei.msg.str(); - to_json(json, ei.errPos); + to_json(json, ei.pos); if (loggerSettings.showTrace.get() && !ei.traces.empty()) { nlohmann::json traces = nlohmann::json::array(); diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 183f2d8e1..9e81132e3 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -120,6 +120,17 @@ public: { } }; +/** + * A variadic template that does nothing. + * + * Useful to call a function with each argument in a parameter pack. + */ +struct nop +{ + template nop(T...) + { } +}; + ActivityId getCurActivity(); void setCurActivity(const ActivityId activityId); diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 5f26fa67b..f8ec7fc6b 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -6,6 +6,38 @@ namespace nix { +PosixSourceAccessor::PosixSourceAccessor(std::filesystem::path && root) + : root(std::move(root)) +{ + assert(root.empty() || root.is_absolute()); + displayPrefix = root; +} + +PosixSourceAccessor::PosixSourceAccessor() + : PosixSourceAccessor(std::filesystem::path {}) +{ } + +std::pair PosixSourceAccessor::createAtRoot(const std::filesystem::path & path) +{ + std::filesystem::path path2 = absPath(path.native()); + return { + PosixSourceAccessor { path2.root_path() }, + CanonPath { static_cast(path2.relative_path()) }, + }; +} + +std::filesystem::path PosixSourceAccessor::makeAbsPath(const CanonPath & path) +{ + return root.empty() + ? (std::filesystem::path { path.abs() }) + : path.isRoot() + ? /* Don't append a slash for the root of the accessor, since + it can be a non-directory (e.g. in the case of `fetchTree + { type = "file" }`). */ + root + : root / path.rel(); +} + void PosixSourceAccessor::readFile( const CanonPath & path, Sink & sink, @@ -13,9 +45,11 @@ void PosixSourceAccessor::readFile( { assertNoSymlinks(path); - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC | O_NOFOLLOW); + auto ap = makeAbsPath(path); + + AutoCloseFD fd = open(ap.c_str(), O_RDONLY | O_CLOEXEC | O_NOFOLLOW); if (!fd) - throw SysError("opening file '%1%'", path); + throw SysError("opening file '%1%'", ap.native()); struct stat st; if (fstat(fd.get(), &st) == -1) @@ -46,7 +80,7 @@ void PosixSourceAccessor::readFile( bool PosixSourceAccessor::pathExists(const CanonPath & path) { if (auto parent = path.parent()) assertNoSymlinks(*parent); - return nix::pathExists(path.abs()); + return nix::pathExists(makeAbsPath(path)); } std::optional PosixSourceAccessor::cachedLstat(const CanonPath & path) @@ -60,7 +94,7 @@ std::optional PosixSourceAccessor::cachedLstat(const CanonPath & pa } std::optional st{std::in_place}; - if (::lstat(path.c_str(), &*st)) { + if (::lstat(makeAbsPath(path).c_str(), &*st)) { if (errno == ENOENT || errno == ENOTDIR) st.reset(); else @@ -95,7 +129,7 @@ SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & { assertNoSymlinks(path); DirEntries res; - for (auto & entry : nix::readDirectory(path.abs())) { + for (auto & entry : nix::readDirectory(makeAbsPath(path))) { std::optional type; switch (entry.type) { case DT_REG: type = Type::tRegular; break; @@ -110,12 +144,12 @@ SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & std::string PosixSourceAccessor::readLink(const CanonPath & path) { if (auto parent = path.parent()) assertNoSymlinks(*parent); - return nix::readLink(path.abs()); + return nix::readLink(makeAbsPath(path)); } -std::optional PosixSourceAccessor::getPhysicalPath(const CanonPath & path) +std::optional PosixSourceAccessor::getPhysicalPath(const CanonPath & path) { - return path; + return makeAbsPath(path); } void PosixSourceAccessor::assertNoSymlinks(CanonPath path) diff --git a/src/libutil/posix-source-accessor.hh b/src/libutil/posix-source-accessor.hh index b2bd39805..717c8f017 100644 --- a/src/libutil/posix-source-accessor.hh +++ b/src/libutil/posix-source-accessor.hh @@ -9,6 +9,16 @@ namespace nix { */ struct PosixSourceAccessor : virtual SourceAccessor { + /** + * Optional root path to prefix all operations into the native file + * system. This allows prepending funny things like `C:\` that + * `CanonPath` intentionally doesn't support. + */ + const std::filesystem::path root; + + PosixSourceAccessor(); + PosixSourceAccessor(std::filesystem::path && root); + /** * The most recent mtime seen by lstat(). This is a hack to * support dumpPathAndGetMtime(). Should remove this eventually. @@ -28,7 +38,22 @@ struct PosixSourceAccessor : virtual SourceAccessor std::string readLink(const CanonPath & path) override; - std::optional getPhysicalPath(const CanonPath & path) override; + std::optional getPhysicalPath(const CanonPath & path) override; + + /** + * Create a `PosixSourceAccessor` and `CanonPath` corresponding to + * some native path. + * + * The `PosixSourceAccessor` is rooted as far up the tree as + * possible, (e.g. on Windows it could scoped to a drive like + * `C:\`). This allows more `..` parent accessing to work. + * + * See + * [`std::filesystem::path::root_path`](https://en.cppreference.com/w/cpp/filesystem/path/root_path) + * and + * [`std::filesystem::path::relative_path`](https://en.cppreference.com/w/cpp/filesystem/path/relative_path). + */ + static std::pair createAtRoot(const std::filesystem::path & path); private: @@ -38,6 +63,8 @@ private: void assertNoSymlinks(CanonPath path); std::optional cachedLstat(const CanonPath & path); + + std::filesystem::path makeAbsPath(const CanonPath & path); }; } diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 7fc211491..70c16ff0d 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -448,7 +448,7 @@ Error readError(Source & source) auto msg = readString(source); ErrorInfo info { .level = level, - .msg = hintfmt(msg), + .msg = HintFmt(msg), }; auto havePos = readNum(source); assert(havePos == 0); @@ -457,7 +457,7 @@ Error readError(Source & source) havePos = readNum(source); assert(havePos == 0); info.traces.push_back(Trace { - .hint = hintfmt(readString(source)) + .hint = HintFmt(readString(source)) }); } return Error(std::move(info)); diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index 4f4ff09c1..aff7da09c 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -1,5 +1,7 @@ #pragma once +#include + #include "canon-path.hh" #include "hash.hh" @@ -119,7 +121,7 @@ struct SourceAccessor * possible. This is only possible for filesystems that are * materialized in the root filesystem. */ - virtual std::optional getPhysicalPath(const CanonPath & path) + virtual std::optional getPhysicalPath(const CanonPath & path) { return std::nullopt; } bool operator == (const SourceAccessor & x) const diff --git a/src/libutil/source-path.cc b/src/libutil/source-path.cc index d85b0b7fe..56ae1d699 100644 --- a/src/libutil/source-path.cc +++ b/src/libutil/source-path.cc @@ -35,17 +35,17 @@ void SourcePath::dumpPath( PathFilter & filter) const { return accessor->dumpPath(path, sink, filter); } -std::optional SourcePath::getPhysicalPath() const +std::optional SourcePath::getPhysicalPath() const { return accessor->getPhysicalPath(path); } std::string SourcePath::to_string() const { return accessor->showPath(path); } -SourcePath SourcePath::operator+(const CanonPath & x) const -{ return {accessor, path + x}; } +SourcePath SourcePath::operator / (const CanonPath & x) const +{ return {accessor, path / x}; } -SourcePath SourcePath::operator+(std::string_view c) const -{ return {accessor, path + c}; } +SourcePath SourcePath::operator / (std::string_view c) const +{ return {accessor, path / c}; } bool SourcePath::operator==(const SourcePath & x) const { @@ -62,7 +62,7 @@ bool SourcePath::operator<(const SourcePath & x) const return std::tie(*accessor, path) < std::tie(*x.accessor, x.path); } -SourcePath SourcePath::resolveSymlinks() const +SourcePath SourcePath::resolveSymlinks(SymlinkResolution mode) const { auto res = SourcePath(accessor); @@ -72,6 +72,8 @@ SourcePath SourcePath::resolveSymlinks() const for (auto & c : path) todo.push_back(std::string(c)); + bool resolve_last = mode == SymlinkResolution::Full; + while (!todo.empty()) { auto c = *todo.begin(); todo.pop_front(); @@ -81,14 +83,16 @@ SourcePath SourcePath::resolveSymlinks() const res.path.pop(); else { res.path.push(c); - if (auto st = res.maybeLstat(); st && st->type == InputAccessor::tSymlink) { - if (!linksAllowed--) - throw Error("infinite symlink recursion in path '%s'", path); - auto target = res.readLink(); - res.path.pop(); - if (hasPrefix(target, "/")) - res.path = CanonPath::root; - todo.splice(todo.begin(), tokenizeString>(target, "/")); + if (resolve_last || !todo.empty()) { + if (auto st = res.maybeLstat(); st && st->type == InputAccessor::tSymlink) { + if (!linksAllowed--) + throw Error("infinite symlink recursion in path '%s'", path); + auto target = res.readLink(); + res.path.pop(); + if (hasPrefix(target, "/")) + res.path = CanonPath::root; + todo.splice(todo.begin(), tokenizeString>(target, "/")); + } } } } diff --git a/src/libutil/source-path.hh b/src/libutil/source-path.hh index bf5625ca5..59991c640 100644 --- a/src/libutil/source-path.hh +++ b/src/libutil/source-path.hh @@ -11,6 +11,26 @@ namespace nix { +/** + * Note there is a decent chance this type soon goes away because the problem is solved another way. + * See the discussion in https://github.com/NixOS/nix/pull/9985. + */ +enum class SymlinkResolution { + /** + * Resolve symlinks in the ancestors only. + * + * Only the last component of the result is possibly a symlink. + */ + Ancestors, + + /** + * Resolve symlinks fully, realpath(3)-style. + * + * No component of the result will be a symlink. + */ + Full, +}; + /** * An abstraction for accessing source files during * evaluation. Currently, it's just a wrapper around `CanonPath` that @@ -82,31 +102,35 @@ struct SourcePath * Return the location of this path in the "real" filesystem, if * it has a physical location. */ - std::optional getPhysicalPath() const; + std::optional getPhysicalPath() const; std::string to_string() const; /** * Append a `CanonPath` to this path. */ - SourcePath operator + (const CanonPath & x) const; + SourcePath operator / (const CanonPath & x) const; /** * Append a single component `c` to this path. `c` must not * contain a slash. A slash is implicitly added between this path * and `c`. */ - SourcePath operator+(std::string_view c) const; + SourcePath operator / (std::string_view c) const; + bool operator==(const SourcePath & x) const; bool operator!=(const SourcePath & x) const; bool operator<(const SourcePath & x) const; /** - * Resolve any symlinks in this `SourcePath` (including its - * parents). The result is a `SourcePath` in which no element is a - * symlink. + * Resolve any symlinks in this `SourcePath` according to the + * given resolution mode. + * + * @param mode might only be a temporary solution for this. + * See the discussion in https://github.com/NixOS/nix/pull/9985. */ - SourcePath resolveSymlinks() const; + SourcePath resolveSymlinks( + SymlinkResolution mode = SymlinkResolution::Full) const; }; std::ostream & operator << (std::ostream & str, const SourcePath & path); diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index 187b3e948..3bb6694f8 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -132,4 +132,66 @@ void unpackTarfile(const Path & tarFile, const Path & destDir) extract_archive(archive, destDir); } +time_t unpackTarfileToSink(TarArchive & archive, FileSystemObjectSink & parseSink) +{ + time_t lastModified = 0; + + for (;;) { + // FIXME: merge with extract_archive + struct archive_entry * entry; + int r = archive_read_next_header(archive.archive, &entry); + if (r == ARCHIVE_EOF) break; + auto path = archive_entry_pathname(entry); + if (!path) + throw Error("cannot get archive member name: %s", archive_error_string(archive.archive)); + if (r == ARCHIVE_WARN) + warn(archive_error_string(archive.archive)); + else + archive.check(r); + + lastModified = std::max(lastModified, archive_entry_mtime(entry)); + + switch (archive_entry_filetype(entry)) { + + case AE_IFDIR: + parseSink.createDirectory(path); + break; + + case AE_IFREG: { + parseSink.createRegularFile(path, [&](auto & crf) { + if (archive_entry_mode(entry) & S_IXUSR) + crf.isExecutable(); + + while (true) { + std::vector buf(128 * 1024); + auto n = archive_read_data(archive.archive, buf.data(), buf.size()); + if (n < 0) + throw Error("cannot read file '%s' from tarball", path); + if (n == 0) break; + crf(std::string_view { + (const char *) buf.data(), + (size_t) n, + }); + } + }); + + break; + } + + case AE_IFLNK: { + auto target = archive_entry_symlink(entry); + + parseSink.createSymlink(path, target); + + break; + } + + default: + throw Error("file '%s' in tarball has unsupported file type", path); + } + } + + return lastModified; +} + } diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh index 237d18c31..6a9c42149 100644 --- a/src/libutil/tarfile.hh +++ b/src/libutil/tarfile.hh @@ -2,6 +2,7 @@ ///@file #include "serialise.hh" +#include "fs-sink.hh" #include namespace nix { @@ -29,4 +30,6 @@ void unpackTarfile(Source & source, const Path & destDir); void unpackTarfile(const Path & tarFile, const Path & destDir); +time_t unpackTarfileToSink(TarArchive & archive, FileSystemObjectSink & parseSink); + } diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 549adfbf7..a372e4b1c 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -299,7 +299,7 @@ static void main_nix_build(int argc, char * * argv) else for (auto i : left) { if (fromArgs) - exprs.push_back(state->parseExprFromString(std::move(i), state->rootPath(CanonPath::fromCwd()))); + exprs.push_back(state->parseExprFromString(std::move(i), state->rootPath("."))); else { auto absolute = i; try { @@ -400,7 +400,7 @@ static void main_nix_build(int argc, char * * argv) try { auto expr = state->parseExprFromString( "(import {}).bashInteractive", - state->rootPath(CanonPath::fromCwd())); + state->rootPath(".")); Value v; state->eval(expr, v); diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 79db78236..48553fa31 100644 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -138,7 +138,7 @@ static void update(const StringSet & channelNames) // Unpack the channel tarballs into the Nix store and install them // into the channels profile. - std::cerr << "unpacking channels...\n"; + std::cerr << "unpacking " << exprs.size() << " channels...\n"; Strings envArgs{ "--profile", profile, "--file", unpackChannelPath, "--install", "--remove-all", "--from-expression" }; for (auto & expr : exprs) envArgs.push_back(std::move(expr)); diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index d5b46c57a..5e3de20c5 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -97,7 +97,7 @@ static bool isNixExpr(const SourcePath & path, struct InputAccessor::Stat & st) { return st.type == InputAccessor::tRegular - || (st.type == InputAccessor::tDirectory && (path + "default.nix").resolveSymlinks().pathExists()); + || (st.type == InputAccessor::tDirectory && (path / "default.nix").resolveSymlinks().pathExists()); } @@ -116,7 +116,7 @@ static void getAllExprs(EvalState & state, are implemented using profiles). */ if (i == "manifest.nix") continue; - auto path2 = (path + i).resolveSymlinks(); + auto path2 = (path / i).resolveSymlinks(); InputAccessor::Stat st; try { @@ -143,7 +143,7 @@ static void getAllExprs(EvalState & state, } /* Load the expression on demand. */ auto vArg = state.allocValue(); - vArg->mkString(path2.path.abs()); + vArg->mkPath(path2); if (seen.size() == maxAttrs) throw Error("too many Nix expressions in directory '%1%'", path); attrs.alloc(attrName).mkApp(&state.getBuiltin("import"), vArg); @@ -413,7 +413,7 @@ static void queryInstSources(EvalState & state, loadSourceExpr(state, *instSource.nixExprPath, vArg); for (auto & i : args) { - Expr * eFun = state.parseExprFromString(i, state.rootPath(CanonPath::fromCwd())); + Expr * eFun = state.parseExprFromString(i, state.rootPath(".")); Value vFun, vTmp; state.eval(eFun, vFun); vTmp.mkApp(&vFun, &vArg); diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 2f9c988d5..8bebe2b9e 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -113,7 +113,7 @@ bool createUserEnv(EvalState & state, PackageInfos & elems, std::string str2 = str.str(); StringSource source { str2 }; state.store->addToStoreFromDump( - source, "env-manifest.nix", TextIngestionMethod {}, HashAlgorithm::SHA256, references); + source, "env-manifest.nix", FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, references); }); /* Get the environment builder expression. */ diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index b9e626aed..86e6f008d 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -168,7 +168,7 @@ static int main_nix_instantiate(int argc, char * * argv) for (auto & i : files) { auto p = state->findFile(i); if (auto fn = p.getPhysicalPath()) - std::cout << fn->abs() << std::endl; + std::cout << fn->native() << std::endl; else throw Error("'%s' has no physical path", p); } @@ -184,7 +184,7 @@ static int main_nix_instantiate(int argc, char * * argv) for (auto & i : files) { Expr * e = fromArgs - ? state->parseExprFromString(i, state->rootPath(CanonPath::fromCwd())) + ? state->parseExprFromString(i, state->rootPath(".")) : state->parseExprFromFile(resolveExprPath(lookupFileArg(*state, i))); processExpr(*state, attrPaths, parseOnly, strict, autoArgs, evalOnly, outputKind, xmlOutputSourceLocation, e); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 40378e123..7c8905da6 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -176,12 +176,11 @@ static void opAdd(Strings opFlags, Strings opArgs) { if (!opFlags.empty()) throw UsageError("unknown flag"); - PosixSourceAccessor accessor; - for (auto & i : opArgs) + for (auto & i : opArgs) { + auto [accessor, canonPath] = PosixSourceAccessor::createAtRoot(i); cout << fmt("%s\n", store->printStorePath(store->addToStore( - std::string(baseNameOf(i)), - accessor, - CanonPath::fromCwd(i)))); + std::string(baseNameOf(i)), accessor, canonPath))); + } } @@ -201,14 +200,15 @@ static void opAddFixed(Strings opFlags, Strings opArgs) HashAlgorithm hashAlgo = parseHashAlgo(opArgs.front()); opArgs.pop_front(); - PosixSourceAccessor accessor; - for (auto & i : opArgs) + for (auto & i : opArgs) { + auto [accessor, canonPath] = PosixSourceAccessor::createAtRoot(i); std::cout << fmt("%s\n", store->printStorePath(store->addToStoreSlow( baseNameOf(i), accessor, - CanonPath::fromCwd(i), + canonPath, method, hashAlgo).path)); + } } @@ -555,7 +555,7 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) HashResult hash = hashPath( *store->getFSAccessor(false), CanonPath { store->printStorePath(info->path) }, - FileIngestionMethod::Recursive, HashAlgorithm::SHA256); + FileSerialisationMethod::Recursive, HashAlgorithm::SHA256); info->narHash = hash.first; info->narSize = hash.second; } @@ -950,8 +950,8 @@ static void opServe(Strings opFlags, Strings opArgs) store->buildPaths(toDerivedPaths(paths)); out << 0; } catch (Error & e) { - assert(e.status); - out << e.status << e.msg(); + assert(e.info().status); + out << e.info().status << e.msg(); } break; } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 7c534517d..02154715f 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -2,21 +2,12 @@ #include "common-args.hh" #include "store-api.hh" #include "archive.hh" +#include "git.hh" #include "posix-source-accessor.hh" +#include "misc-store-flags.hh" using namespace nix; -static FileIngestionMethod parseIngestionMethod(std::string_view input) -{ - if (input == "flat") { - return FileIngestionMethod::Flat; - } else if (input == "nar") { - return FileIngestionMethod::Recursive; - } else { - throw UsageError("Unknown hash mode '%s', expect `flat` or `nar`"); - } -} - struct CmdAddToStore : MixDryRun, StoreCommand { Path path; @@ -37,32 +28,16 @@ struct CmdAddToStore : MixDryRun, StoreCommand .handler = {&namePart}, }); - addFlag({ - .longName = "mode", - .description = R"( - How to compute the hash of the input. - One of: + addFlag(flag::contentAddressMethod(&caMethod)); - - `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function. - - - `flat`: Assumes that the input is a single file and directly passes it to the hash function; - )", - .labels = {"hash-mode"}, - .handler = {[this](std::string s) { - this->caMethod = parseIngestionMethod(s); - }}, - }); - - addFlag(Flag::mkHashAlgoFlag(&hashAlgo)); + addFlag(flag::hashAlgo(&hashAlgo)); } void run(ref store) override { if (!namePart) namePart = baseNameOf(path); - PosixSourceAccessor accessor; - - auto path2 = CanonPath::fromCwd(path); + auto [accessor, path2] = PosixSourceAccessor::createAtRoot(path); auto storePath = dryRun ? store->computeStorePath( @@ -76,7 +51,6 @@ struct CmdAddToStore : MixDryRun, StoreCommand struct CmdAdd : CmdAddToStore { - std::string description() override { return "Add a file or directory to the Nix store"; diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index 4dada8e0e..8afcbe982 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -377,7 +377,7 @@ static void daemonLoop(std::optional forceTrustClientOpt) } catch (Error & error) { auto ei = error.info(); // FIXME: add to trace? - ei.msg = hintfmt("error processing connection: %1%", ei.msg.str()); + ei.msg = HintFmt("error processing connection: %1%", ei.msg.str()); logError(ei); } } diff --git a/src/nix/derivation-add.md b/src/nix/derivation-add.md index d9b8467df..331cbdd88 100644 --- a/src/nix/derivation-add.md +++ b/src/nix/derivation-add.md @@ -14,6 +14,6 @@ a Nix expression evaluates. `nix derivation add` takes a single derivation in the following format: -{{#include ../../json/derivation.md}} +{{#include ../../protocols/json/derivation.md}} )"" diff --git a/src/nix/derivation-show.md b/src/nix/derivation-show.md index 884f1adc6..2437ea08f 100644 --- a/src/nix/derivation-show.md +++ b/src/nix/derivation-show.md @@ -52,6 +52,6 @@ By default, this command only shows top-level derivations, but with [store path]: @docroot@/glossary.md#gloss-store-path -{{#include ../../json/derivation.md}} +{{#include ../../protocols/json/derivation.md}} )"" diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 1f2891378..c1842f2d5 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -226,7 +226,7 @@ static StorePath getDerivationEnvironment(ref store, ref evalStore auto getEnvShPath = ({ StringSource source { getEnvSh }; evalStore->addToStoreFromDump( - source, "get-env.sh", TextIngestionMethod {}, HashAlgorithm::SHA256, {}); + source, "get-env.sh", FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, {}); }); drv.args = {store->printStorePath(getEnvShPath)}; @@ -354,7 +354,7 @@ struct Common : InstallableCommand, MixProfile for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"}) out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i); - out << "eval \"$shellHook\"\n"; + out << "eval \"${shellHook:-}\"\n"; auto script = out.str(); diff --git a/src/nix/eval.cc b/src/nix/eval.cc index a89fa7412..2044c8c2b 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -66,7 +66,7 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption if (apply) { auto vApply = state->allocValue(); - state->eval(state->parseExprFromString(*apply, state->rootPath(CanonPath::fromCwd())), *vApply); + state->eval(state->parseExprFromString(*apply, state->rootPath(".")), *vApply); auto vRes = state->allocValue(); state->callFunction(*vApply, *v, *vRes, noPos); v = vRes; @@ -98,13 +98,13 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption } catch (Error & e) { e.addTrace( state->positions[attr.pos], - hintfmt("while evaluating the attribute '%s'", name)); + HintFmt("while evaluating the attribute '%s'", name)); throw; } } } else - throw TypeError("value at '%s' is not a string or an attribute set", state->positions[pos]); + state->error("value at '%s' is not a string or an attribute set", state->positions[pos]).debugThrow(); }; recurse(*v, pos, *writeTo); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 0e34bd76a..131589f35 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -224,7 +224,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON if (auto lastModified = flake.lockedRef.input.getLastModified()) j["lastModified"] = *lastModified; j["path"] = store->printStorePath(flake.storePath); - j["locks"] = lockedFlake.lockFile.toJSON(); + j["locks"] = lockedFlake.lockFile.toJSON().first; logger->cout("%s", j.dump()); } else { logger->cout( @@ -411,7 +411,7 @@ struct CmdFlakeCheck : FlakeCommand return storePath; } } catch (Error & e) { - e.addTrace(resolve(pos), hintfmt("while checking the derivation '%s'", attrPath)); + e.addTrace(resolve(pos), HintFmt("while checking the derivation '%s'", attrPath)); reportError(e); } return std::nullopt; @@ -430,7 +430,7 @@ struct CmdFlakeCheck : FlakeCommand } #endif } catch (Error & e) { - e.addTrace(resolve(pos), hintfmt("while checking the app definition '%s'", attrPath)); + e.addTrace(resolve(pos), HintFmt("while checking the app definition '%s'", attrPath)); reportError(e); } }; @@ -454,7 +454,7 @@ struct CmdFlakeCheck : FlakeCommand // FIXME: if we have a 'nixpkgs' input, use it to // evaluate the overlay. } catch (Error & e) { - e.addTrace(resolve(pos), hintfmt("while checking the overlay '%s'", attrPath)); + e.addTrace(resolve(pos), HintFmt("while checking the overlay '%s'", attrPath)); reportError(e); } }; @@ -465,7 +465,7 @@ struct CmdFlakeCheck : FlakeCommand fmt("checking NixOS module '%s'", attrPath)); state->forceValue(v, pos); } catch (Error & e) { - e.addTrace(resolve(pos), hintfmt("while checking the NixOS module '%s'", attrPath)); + e.addTrace(resolve(pos), HintFmt("while checking the NixOS module '%s'", attrPath)); reportError(e); } }; @@ -491,7 +491,7 @@ struct CmdFlakeCheck : FlakeCommand } } catch (Error & e) { - e.addTrace(resolve(pos), hintfmt("while checking the Hydra jobset '%s'", attrPath)); + e.addTrace(resolve(pos), HintFmt("while checking the Hydra jobset '%s'", attrPath)); reportError(e); } }; @@ -506,7 +506,7 @@ struct CmdFlakeCheck : FlakeCommand if (!state->isDerivation(*vToplevel)) throw Error("attribute 'config.system.build.toplevel' is not a derivation"); } catch (Error & e) { - e.addTrace(resolve(pos), hintfmt("while checking the NixOS configuration '%s'", attrPath)); + e.addTrace(resolve(pos), HintFmt("while checking the NixOS configuration '%s'", attrPath)); reportError(e); } }; @@ -540,7 +540,7 @@ struct CmdFlakeCheck : FlakeCommand throw Error("template '%s' has unsupported attribute '%s'", attrPath, name); } } catch (Error & e) { - e.addTrace(resolve(pos), hintfmt("while checking the template '%s'", attrPath)); + e.addTrace(resolve(pos), HintFmt("while checking the template '%s'", attrPath)); reportError(e); } }; @@ -554,7 +554,7 @@ struct CmdFlakeCheck : FlakeCommand throw Error("bundler must be a function"); // TODO: check types of inputs/outputs? } catch (Error & e) { - e.addTrace(resolve(pos), hintfmt("while checking the template '%s'", attrPath)); + e.addTrace(resolve(pos), HintFmt("while checking the template '%s'", attrPath)); reportError(e); } }; @@ -774,7 +774,7 @@ struct CmdFlakeCheck : FlakeCommand warn("unknown flake output '%s'", name); } catch (Error & e) { - e.addTrace(resolve(pos), hintfmt("while checking flake output '%s'", name)); + e.addTrace(resolve(pos), HintFmt("while checking flake output '%s'", name)); reportError(e); } }); @@ -848,10 +848,10 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand auto templateDir = templateDirAttr->getString(); if (!store->isInStore(templateDir)) - throw TypeError( + evalState->error( "'%s' was not found in the Nix store\n" "If you've set '%s' to a string, try using a path instead.", - templateDir, templateDirAttr->getAttrPathStr()); + templateDir, templateDirAttr->getAttrPathStr()).debugThrow(); std::vector changedFiles; std::vector conflictedFiles; @@ -1321,7 +1321,7 @@ struct CmdFlakeShow : FlakeCommand, MixJSON { auto aType = visitor.maybeGetAttr("type"); if (!aType || aType->getString() != "app") - throw EvalError("not an app definition"); + state->error("not an app definition").debugThrow(); if (json) { j.emplace("type", "app"); } else { diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 4837891c6..f849bf0cf 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -5,12 +5,14 @@ #include "shared.hh" #include "references.hh" #include "archive.hh" +#include "git.hh" #include "posix-source-accessor.hh" +#include "misc-store-flags.hh" using namespace nix; /** - * Base for `nix hash file` (deprecated), `nix hash path` and `nix-hash` (legacy). + * Base for `nix hash path`, `nix hash file` (deprecated), and `nix-hash` (legacy). * * Deprecation Issue: https://github.com/NixOS/nix/issues/8876 */ @@ -19,12 +21,21 @@ struct CmdHashBase : Command FileIngestionMethod mode; HashFormat hashFormat = HashFormat::SRI; bool truncate = false; - HashAlgorithm ha = HashAlgorithm::SHA256; + HashAlgorithm hashAlgo = HashAlgorithm::SHA256; std::vector paths; std::optional modulus; explicit CmdHashBase(FileIngestionMethod mode) : mode(mode) { + expectArgs({ + .label = "paths", + .handler = {&paths}, + .completer = completePath + }); + + // FIXME The following flags should be deprecated, but we don't + // yet have a mechanism for that. + addFlag({ .longName = "sri", .description = "Print the hash in SRI format.", @@ -49,31 +60,18 @@ struct CmdHashBase : Command .handler = {&hashFormat, HashFormat::Base16}, }); - addFlag(Flag::mkHashAlgoFlag("type", &ha)); - - #if 0 - addFlag({ - .longName = "modulo", - .description = "Compute the hash modulo the specified string.", - .labels = {"modulus"}, - .handler = {&modulus}, - }); - #endif\ - - expectArgs({ - .label = "paths", - .handler = {&paths}, - .completer = completePath - }); + addFlag(flag::hashAlgo("type", &hashAlgo)); } std::string description() override { switch (mode) { case FileIngestionMethod::Flat: - return "print cryptographic hash of a regular file"; + return "print cryptographic hash of a regular file"; case FileIngestionMethod::Recursive: return "print cryptographic hash of the NAR serialisation of a path"; + case FileIngestionMethod::Git: + return "print cryptographic hash of the Git serialisation of a path"; default: assert(false); }; @@ -82,32 +80,94 @@ struct CmdHashBase : Command void run() override { for (auto path : paths) { + auto makeSink = [&]() -> std::unique_ptr { + if (modulus) + return std::make_unique(hashAlgo, *modulus); + else + return std::make_unique(hashAlgo); + }; - std::unique_ptr hashSink; - if (modulus) - hashSink = std::make_unique(ha, *modulus); - else - hashSink = std::make_unique(ha); + auto [accessor_, canonPath] = PosixSourceAccessor::createAtRoot(path); + auto & accessor = accessor_; + Hash h { HashAlgorithm::SHA256 }; // throwaway def to appease C++ + switch (mode) { + case FileIngestionMethod::Flat: + case FileIngestionMethod::Recursive: + { + auto hashSink = makeSink(); + dumpPath(accessor, canonPath, *hashSink, (FileSerialisationMethod) mode); + h = hashSink->finish().first; + break; + } + case FileIngestionMethod::Git: { + std::function hook; + hook = [&](const CanonPath & path) -> git::TreeEntry { + auto hashSink = makeSink(); + auto mode = dump(accessor, path, *hashSink, hook); + auto hash = hashSink->finish().first; + return { + .mode = mode, + .hash = hash, + }; + }; + h = hook(canonPath).hash; + break; + } + } - PosixSourceAccessor accessor; - dumpPath(accessor, CanonPath::fromCwd(path), *hashSink, mode); - - Hash h = hashSink->finish().first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); logger->cout(h.to_string(hashFormat, hashFormat == HashFormat::SRI)); } } }; +/** + * `nix hash path` + */ +struct CmdHashPath : CmdHashBase +{ + CmdHashPath() + : CmdHashBase(FileIngestionMethod::Recursive) + { + addFlag(flag::hashAlgo("algo", &hashAlgo)); + addFlag(flag::fileIngestionMethod(&mode)); + addFlag(flag::hashFormatWithDefault("format", &hashFormat)); + #if 0 + addFlag({ + .longName = "modulo", + .description = "Compute the hash modulo the specified string.", + .labels = {"modulus"}, + .handler = {&modulus}, + }); + #endif + } +}; + +/** + * For deprecated `nix hash file` + * + * Deprecation Issue: https://github.com/NixOS/nix/issues/8876 + */ +struct CmdHashFile : CmdHashBase +{ + CmdHashFile() + : CmdHashBase(FileIngestionMethod::Flat) + { + } +}; + +/** + * For deprecated `nix hash to-*` + */ struct CmdToBase : Command { HashFormat hashFormat; - std::optional ht; + std::optional hashAlgo; std::vector args; CmdToBase(HashFormat hashFormat) : hashFormat(hashFormat) { - addFlag(Flag::mkHashAlgoOptFlag("type", &ht)); + addFlag(flag::hashAlgoOpt("type", &hashAlgo)); expectArgs("strings", &args); } @@ -124,7 +184,7 @@ struct CmdToBase : Command { warn("The old format conversion sub commands of `nix hash` where deprecated in favor of `nix hash convert`."); for (auto s : args) - logger->cout(Hash::parseAny(s, ht).to_string(hashFormat, hashFormat == HashFormat::SRI)); + logger->cout(Hash::parseAny(s, hashAlgo).to_string(hashFormat, hashFormat == HashFormat::SRI)); } }; @@ -139,9 +199,9 @@ struct CmdHashConvert : Command std::vector hashStrings; CmdHashConvert(): to(HashFormat::SRI) { - addFlag(Args::Flag::mkHashFormatOptFlag("from", &from)); - addFlag(Args::Flag::mkHashFormatFlagWithDefault("to", &to)); - addFlag(Args::Flag::mkHashAlgoOptFlag(&algo)); + addFlag(flag::hashFormatOpt("from", &from)); + addFlag(flag::hashFormatWithDefault("to", &to)); + addFlag(flag::hashAlgoOpt(&algo)); expectArgs({ .label = "hashes", .handler = {&hashStrings}, @@ -181,8 +241,8 @@ struct CmdHash : NixMultiCommand "hash", { {"convert", []() { return make_ref();}}, - {"file", []() { return make_ref(FileIngestionMethod::Flat);; }}, - {"path", []() { return make_ref(FileIngestionMethod::Recursive); }}, + {"path", []() { return make_ref(); }}, + {"file", []() { return make_ref(); }}, {"to-base16", []() { return make_ref(HashFormat::Base16); }}, {"to-base32", []() { return make_ref(HashFormat::Nix32); }}, {"to-base64", []() { return make_ref(HashFormat::Base64); }}, @@ -206,7 +266,7 @@ static int compatNixHash(int argc, char * * argv) // Wait until `nix hash convert` is not hidden behind experimental flags anymore. // warn("`nix-hash` has been deprecated in favor of `nix hash convert`."); - std::optional ha; + std::optional hashAlgo; bool flat = false; HashFormat hashFormat = HashFormat::Base16; bool truncate = false; @@ -226,7 +286,7 @@ static int compatNixHash(int argc, char * * argv) else if (*arg == "--truncate") truncate = true; else if (*arg == "--type") { std::string s = getArg(*arg, arg, end); - ha = parseHashAlgo(s); + hashAlgo = parseHashAlgo(s); } else if (*arg == "--to-base16") { op = opTo; @@ -253,8 +313,8 @@ static int compatNixHash(int argc, char * * argv) if (op == opHash) { CmdHashBase cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive); - if (!ha.has_value()) ha = HashAlgorithm::MD5; - cmd.ha = ha.value(); + if (!hashAlgo.has_value()) hashAlgo = HashAlgorithm::MD5; + cmd.hashAlgo = hashAlgo.value(); cmd.hashFormat = hashFormat; cmd.truncate = truncate; cmd.paths = ss; @@ -264,7 +324,7 @@ static int compatNixHash(int argc, char * * argv) else { CmdToBase cmd(hashFormat); cmd.args = ss; - if (ha.has_value()) cmd.ht = ha; + if (hashAlgo.has_value()) cmd.hashAlgo = hashAlgo; cmd.run(); } diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 231456c9c..63f97f2d3 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -72,7 +72,7 @@ struct MixLs : virtual Args, MixJSON if (st.type == SourceAccessor::Type::tDirectory && !showDirectory) { auto names = accessor->readDirectory(curPath); for (auto & [name, type] : names) - showFile(curPath + name, relPath + "/" + name); + showFile(curPath / name, relPath + "/" + name); } else showFile(curPath, relPath); }; diff --git a/src/nix/main.cc b/src/nix/main.cc index 39c04069b..5af5f2e41 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -32,6 +32,24 @@ void chrootHelper(int argc, char * * argv); namespace nix { +static bool haveProxyEnvironmentVariables() +{ + static const std::vector proxyVariables = { + "http_proxy", + "https_proxy", + "ftp_proxy", + "HTTP_PROXY", + "HTTPS_PROXY", + "FTP_PROXY" + }; + for (auto & proxyVariable: proxyVariables) { + if (getEnv(proxyVariable).has_value()) { + return true; + } + } + return false; +} + /* Check if we have a non-loopback/link-local network interface. */ static bool haveInternet() { @@ -55,6 +73,8 @@ static bool haveInternet() } } + if (haveProxyEnvironmentVariables()) return true; + return false; } diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 84b79ea28..fabec5d88 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -10,6 +10,7 @@ #include "eval-inline.hh" #include "legacy.hh" #include "posix-source-accessor.hh" +#include "misc-store-flags.hh" #include @@ -123,10 +124,9 @@ std::tuple prefetchFile( Activity act(*logger, lvlChatty, actUnknown, fmt("adding '%s' to the store", url)); - PosixSourceAccessor accessor; + auto [accessor, canonPath] = PosixSourceAccessor::createAtRoot(tmpFile); auto info = store->addToStoreSlow( - *name, - accessor, CanonPath::fromCwd(tmpFile), + *name, accessor, canonPath, ingestionMethod, hashAlgo, {}, expectedHash); storePath = info.path; assert(info.ca); @@ -285,7 +285,7 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON }} }); - addFlag(Flag::mkHashAlgoFlag("hash-type", &hashAlgo)); + addFlag(flag::hashAlgo("hash-type", &hashAlgo)); addFlag({ .longName = "executable", diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 812e703b4..d39a24d36 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -101,6 +101,15 @@ struct ProfileElement } }; +std::string getNameFromElement(const ProfileElement & element) +{ + std::optional result = std::nullopt; + if (element.source) { + result = getNameFromURL(parseURL(element.source->to_string())); + } + return result.value_or(element.identifier()); +} + struct ProfileManifest { using ProfileElementName = std::string; @@ -189,12 +198,8 @@ struct ProfileManifest void addElement(ProfileElement element) { - auto name = - element.source - ? getNameFromURL(parseURL(element.source->to_string())) - : std::nullopt; - auto name2 = name ? *name : element.identifier(); - addElement(name2, std::move(element)); + auto name = getNameFromElement(element); + addElement(name, std::move(element)); } nlohmann::json toJSON(Store & store) const @@ -390,7 +395,26 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile element.updateStorePaths(getEvalStore(), store, res); - manifest.addElement(std::move(element)); + auto elementName = getNameFromElement(element); + + // Check if the element already exists. + auto existingPair = manifest.elements.find(elementName); + if (existingPair != manifest.elements.end()) { + auto existingElement = existingPair->second; + auto existingSource = existingElement.source; + auto elementSource = element.source; + if (existingSource + && elementSource + && existingElement.priority == element.priority + && existingSource->originalRef == elementSource->originalRef + && existingSource->attrPath == elementSource->attrPath + ) { + warn("'%s' is already installed", elementName); + continue; + } + } + + manifest.addElement(elementName, std::move(element)); } try { @@ -400,13 +424,13 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile // See https://github.com/NixOS/nix/compare/3efa476c5439f8f6c1968a6ba20a31d1239c2f04..1fe5d172ece51a619e879c4b86f603d9495cc102 auto findRefByFilePath = [&](Iterator begin, Iterator end) { for (auto it = begin; it != end; it++) { - auto & profileElement = it->second; + auto & [name, profileElement] = *it; for (auto & storePath : profileElement.storePaths) { if (conflictError.fileA.starts_with(store->printStorePath(storePath))) { - return std::pair(conflictError.fileA, profileElement.toInstallables(*store)); + return std::tuple(conflictError.fileA, name, profileElement.toInstallables(*store)); } if (conflictError.fileB.starts_with(store->printStorePath(storePath))) { - return std::pair(conflictError.fileB, profileElement.toInstallables(*store)); + return std::tuple(conflictError.fileB, name, profileElement.toInstallables(*store)); } } } @@ -415,9 +439,9 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile // There are 2 conflicting files. We need to find out which one is from the already installed package and // which one is the package that is the new package that is being installed. // The first matching package is the one that was already installed (original). - auto [originalConflictingFilePath, originalConflictingRefs] = findRefByFilePath(manifest.elements.begin(), manifest.elements.end()); + auto [originalConflictingFilePath, originalEntryName, originalConflictingRefs] = findRefByFilePath(manifest.elements.begin(), manifest.elements.end()); // The last matching package is the one that was going to be installed (new). - auto [newConflictingFilePath, newConflictingRefs] = findRefByFilePath(manifest.elements.rbegin(), manifest.elements.rend()); + auto [newConflictingFilePath, newEntryName, newConflictingRefs] = findRefByFilePath(manifest.elements.rbegin(), manifest.elements.rend()); throw Error( "An existing package already provides the following file:\n" @@ -443,7 +467,7 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile " nix profile install %4% --priority %7%\n", originalConflictingFilePath, newConflictingFilePath, - concatStringsSep(" ", originalConflictingRefs), + originalEntryName, concatStringsSep(" ", newConflictingRefs), conflictError.priority, conflictError.priority - 1, diff --git a/src/nix/run.cc b/src/nix/run.cc index 9bca5b9d0..e86837679 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -124,7 +124,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment if (true) pathAdditions.push_back(store->printStorePath(path) + "/bin"); - auto propPath = CanonPath(store->printStorePath(path)) + "nix-support" + "propagated-user-env-packages"; + auto propPath = CanonPath(store->printStorePath(path)) / "nix-support" / "propagated-user-env-packages"; if (auto st = accessor->maybeLstat(propPath); st && st->type == SourceAccessor::tRegular) { for (auto & p : tokenizeString(accessor->readFile(propPath))) todo.push(store->parseStorePath(p)); diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index aecf65922..e299585ff 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -225,7 +225,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions if (st->type == SourceAccessor::Type::tDirectory) { auto names = accessor->readDirectory(p); for (auto & [name, type] : names) - visitPath(p + name); + visitPath(p / name); } else if (st->type == SourceAccessor::Type::tRegular) { diff --git a/tests/functional/add.sh b/tests/functional/add.sh index 762e01dbe..a4bb0e225 100644 --- a/tests/functional/add.sh +++ b/tests/functional/add.sh @@ -45,3 +45,8 @@ clearStore [[ "$path1" == "$path2" ]] path4=$(nix store add --mode flat --hash-algo sha1 ./dummy) ) +( + path1=$(nix store add --mode text ./dummy) + path2=$(nix eval --impure --raw --expr 'builtins.toFile "dummy" (builtins.readFile ./dummy)') + [[ "$path1" == "$path2" ]] +) diff --git a/tests/functional/bad.tar.xz b/tests/functional/bad.tar.xz deleted file mode 100644 index 250a5ad1a..000000000 Binary files a/tests/functional/bad.tar.xz and /dev/null differ diff --git a/tests/functional/debugger.sh b/tests/functional/debugger.sh new file mode 100644 index 000000000..63d88cbf3 --- /dev/null +++ b/tests/functional/debugger.sh @@ -0,0 +1,13 @@ +source common.sh + +clearStore + +# regression #9932 +echo ":env" | expect 1 nix eval --debugger --expr '(_: throw "oh snap") 42' +echo ":env" | expect 1 nix eval --debugger --expr ' + let x.a = 1; in + with x; + (_: builtins.seq x.a (throw "oh snap")) x.a +' >debugger-test-out +grep -P 'with: .*a' debugger-test-out +grep -P 'static: .*x' debugger-test-out diff --git a/tests/functional/experimental-features.sh b/tests/functional/experimental-features.sh index 9ee4a53d4..12112b293 100644 --- a/tests/functional/experimental-features.sh +++ b/tests/functional/experimental-features.sh @@ -31,17 +31,19 @@ source common.sh NIX_CONFIG=' experimental-features = nix-command accept-flake-config = true -' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr -grepQuiet "false" $TEST_ROOT/stdout +' expect 1 nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +[[ $(cat $TEST_ROOT/stdout) = '' ]] grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" $TEST_ROOT/stderr +grepQuiet "error: could not find setting 'accept-flake-config'" $TEST_ROOT/stderr # 'flakes' experimental-feature is disabled after, ignore and warn NIX_CONFIG=' accept-flake-config = true experimental-features = nix-command -' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr -grepQuiet "false" $TEST_ROOT/stdout +' expect 1 nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +[[ $(cat $TEST_ROOT/stdout) = '' ]] grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" $TEST_ROOT/stderr +grepQuiet "error: could not find setting 'accept-flake-config'" $TEST_ROOT/stderr # 'flakes' experimental-feature is enabled before, process NIX_CONFIG=' diff --git a/tests/functional/fetchGit.sh b/tests/functional/fetchGit.sh index c6a482035..3f2d0d5fb 100644 --- a/tests/functional/fetchGit.sh +++ b/tests/functional/fetchGit.sh @@ -30,7 +30,10 @@ echo hello >> $TEST_ROOT/worktree/hello rev2=$(git -C $repo rev-parse HEAD) git -C $repo tag -a tag2 -m tag2 -# Fetch a worktree +# Check whether fetching in read-only mode works. +nix-instantiate --eval -E "builtins.readFile ((builtins.fetchGit file://$TEST_ROOT/worktree) + \"/hello\") == \"utrecht\\n\"" + +# Fetch a worktree. unset _NIX_FORCE_HTTP path0=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$TEST_ROOT/worktree).outPath") path0_=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; url = file://$TEST_ROOT/worktree; }).outPath") @@ -67,7 +70,7 @@ path2=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \" [[ $(nix eval --raw --expr "builtins.readFile (fetchGit { url = file://$repo; rev = \"$rev2\"; } + \"/hello\")") = world ]] # But without a hash, it fails -expectStderr 1 nix eval --expr 'builtins.fetchGit "file:///foo"' | grepQuiet "'fetchGit' requires a locked input" +expectStderr 1 nix eval --expr 'builtins.fetchGit "file:///foo"' | grepQuiet "'fetchGit' will not fetch unlocked input" # Fetch again. This should be cached. mv $repo ${repo}-tmp @@ -208,7 +211,7 @@ path6=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; ur [[ $path3 = $path6 ]] [[ $(nix eval --impure --expr "(builtins.fetchTree { type = \"git\"; url = \"file://$TEST_ROOT/shallow\"; ref = \"dev\"; shallow = true; }).revCount or 123") == 123 ]] -expectStderr 1 nix eval --expr 'builtins.fetchTree { type = "git"; url = "file:///foo"; }' | grepQuiet "'fetchTree' requires a locked input" +expectStderr 1 nix eval --expr 'builtins.fetchTree { type = "git"; url = "file:///foo"; }' | grepQuiet "'fetchTree' will not fetch unlocked input" # Explicit ref = "HEAD" should work, and produce the same outPath as without ref path7=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"HEAD\"; }).outPath") @@ -268,3 +271,28 @@ git -C "$repo" add hello .gitignore git -C "$repo" commit -m 'Bla1' cd "$repo" path11=$(nix eval --impure --raw --expr "(builtins.fetchGit ./.).outPath") + +# Test a workdir with no commits. +empty="$TEST_ROOT/empty" +git init "$empty" + +emptyAttrs='{ lastModified = 0; lastModifiedDate = "19700101000000"; narHash = "sha256-pQpattmS9VmO3ZIQUFn66az8GSmB4IvYhTTCFn6SUmo="; rev = "0000000000000000000000000000000000000000"; revCount = 0; shortRev = "0000000"; submodules = false; }' + +[[ $(nix eval --impure --expr "builtins.removeAttrs (builtins.fetchGit $empty) [\"outPath\"]") = $emptyAttrs ]] + +echo foo > "$empty/x" + +[[ $(nix eval --impure --expr "builtins.removeAttrs (builtins.fetchGit $empty) [\"outPath\"]") = $emptyAttrs ]] + +git -C "$empty" add x + +[[ $(nix eval --impure --expr "builtins.removeAttrs (builtins.fetchGit $empty) [\"outPath\"]") = '{ lastModified = 0; lastModifiedDate = "19700101000000"; narHash = "sha256-wzlAGjxKxpaWdqVhlq55q5Gxo4Bf860+kLeEa/v02As="; rev = "0000000000000000000000000000000000000000"; revCount = 0; shortRev = "0000000"; submodules = false; }' ]] + +# Test a repo with an empty commit. +git -C "$empty" rm -f x + +git -C "$empty" config user.email "foobar@example.com" +git -C "$empty" config user.name "Foobar" +git -C "$empty" commit --allow-empty --allow-empty-message --message "" + +nix eval --impure --expr "let attrs = builtins.fetchGit $empty; in assert attrs.lastModified != 0; assert attrs.rev != \"0000000000000000000000000000000000000000\"; assert attrs.revCount == 1; true" diff --git a/tests/functional/fetchTree-file.sh b/tests/functional/fetchTree-file.sh index 6395c133d..be698ea35 100644 --- a/tests/functional/fetchTree-file.sh +++ b/tests/functional/fetchTree-file.sh @@ -14,6 +14,7 @@ test_fetch_file () { tree = builtins.fetchTree { type = "file"; url = "file://$PWD/test_input"; }; in assert (tree.narHash == "$input_hash"); + assert builtins.readFile tree == "foo\n"; tree EOF } diff --git a/tests/functional/fetchurl.sh b/tests/functional/fetchurl.sh index 8cd40c09f..a3620f52b 100644 --- a/tests/functional/fetchurl.sh +++ b/tests/functional/fetchurl.sh @@ -78,3 +78,9 @@ outPath=$(nix-build -vvvvv --expr 'import ' --argstr url file: test -x $outPath/fetchurl.sh test -L $outPath/symlink + +# Make sure that *not* passing a outputHash fails. +requireDaemonNewerThan "2.20" +expected=100 +if [[ -v NIX_DAEMON_PACKAGE ]]; then expected=1; fi # work around the daemon not returning a 100 status correctly +expectStderr $expected nix-build --expr '{ url }: builtins.derivation { name = "nix-cache-info"; system = "x86_64-linux"; builder = "builtin:fetchurl"; inherit url; outputHashMode = "flat"; }' --argstr url file://$narxz 2>&1 | grep 'must be a fixed-output or impure derivation' diff --git a/tests/functional/git-hashing/common.sh b/tests/functional/git-hashing/common.sh new file mode 100644 index 000000000..572cea438 --- /dev/null +++ b/tests/functional/git-hashing/common.sh @@ -0,0 +1,11 @@ +source ../common.sh + +clearStore +clearCache + +# Need backend to support git-hashing too +requireDaemonNewerThan "2.19" + +enableFeatures "git-hashing" + +restartDaemon diff --git a/tests/functional/git-hashing/local.mk b/tests/functional/git-hashing/local.mk new file mode 100644 index 000000000..ebec01940 --- /dev/null +++ b/tests/functional/git-hashing/local.mk @@ -0,0 +1,7 @@ +git-hashing-tests := \ + $(d)/simple.sh + +install-tests-groups += git-hashing + +clean-files += \ + $(d)/config.nix diff --git a/tests/functional/git-hashing/simple.sh b/tests/functional/git-hashing/simple.sh new file mode 100644 index 000000000..74b0220f8 --- /dev/null +++ b/tests/functional/git-hashing/simple.sh @@ -0,0 +1,58 @@ +source common.sh + +repo="$TEST_ROOT/scratch" +git init "$repo" + +git -C "$repo" config user.email "you@example.com" +git -C "$repo" config user.name "Your Name" + +try () { + hash=$(nix hash path --mode git --format base16 --algo sha1 $TEST_ROOT/hash-path) + [[ "$hash" == "$1" ]] + + git -C "$repo" rm -rf hash-path || true + cp -r "$TEST_ROOT/hash-path" "$TEST_ROOT/scratch/hash-path" + git -C "$repo" add hash-path + git -C "$repo" commit -m "x" + git -C "$repo" status + hash2=$(git -C "$TEST_ROOT/scratch" rev-parse HEAD:hash-path) + [[ "$hash2" = "$1" ]] +} + +# blob +rm -rf $TEST_ROOT/hash-path +echo "Hello World" > $TEST_ROOT/hash-path +try "557db03de997c86a4a028e1ebd3a1ceb225be238" + +# tree with children +rm -rf $TEST_ROOT/hash-path +mkdir $TEST_ROOT/hash-path +echo "Hello World" > $TEST_ROOT/hash-path/hello +echo "Run Hello World" > $TEST_ROOT/hash-path/executable +chmod +x $TEST_ROOT/hash-path/executable +try "e5c0a11a556801a5c9dcf330ca9d7e2c572697f4" + +rm -rf $TEST_ROOT/dummy1 +echo Hello World! > $TEST_ROOT/dummy1 +path1=$(nix store add --mode git --hash-algo sha1 $TEST_ROOT/dummy1) +hash1=$(nix-store -q --hash $path1) +test "$hash1" = "sha256:1brffhvj2c0z6x8qismd43m0iy8dsgfmy10bgg9w11szway2wp9v" + +rm -rf $TEST_ROOT/dummy2 +mkdir -p $TEST_ROOT/dummy2 +echo Hello World! > $TEST_ROOT/dummy2/hello +path2=$(nix store add --mode git --hash-algo sha1 $TEST_ROOT/dummy2) +hash2=$(nix-store -q --hash $path2) +test "$hash2" = "sha256:1vhv7zxam7x277q0y0jcypm7hwhccbzss81vkdgf0ww5sm2am4y0" + +rm -rf $TEST_ROOT/dummy3 +mkdir -p $TEST_ROOT/dummy3 +mkdir -p $TEST_ROOT/dummy3/dir +touch $TEST_ROOT/dummy3/dir/file +echo Hello World! > $TEST_ROOT/dummy3/dir/file +touch $TEST_ROOT/dummy3/dir/executable +chmod +x $TEST_ROOT/dummy3/dir/executable +echo Run Hello World! > $TEST_ROOT/dummy3/dir/executable +path3=$(nix store add --mode git --hash-algo sha1 $TEST_ROOT/dummy3) +hash3=$(nix-store -q --hash $path3) +test "$hash3" = "sha256:08y3nm3mvn9qvskqnf13lfgax5lh73krxz4fcjd5cp202ggpw9nv" diff --git a/tests/functional/hash-convert.sh b/tests/functional/hash-convert.sh new file mode 100644 index 000000000..9b3afc10b --- /dev/null +++ b/tests/functional/hash-convert.sh @@ -0,0 +1,105 @@ +source common.sh + +# Conversion with `nix hash` `nix-hash` and `nix hash convert` +try3() { + # $1 = hash algo + # $2 = expected hash in base16 + # $3 = expected hash in base32 + # $4 = expected hash in base64 + h64=$(nix hash convert --hash-algo "$1" --to base64 "$2") + [ "$h64" = "$4" ] + h64=$(nix-hash --type "$1" --to-base64 "$2") + [ "$h64" = "$4" ] + # Deprecated experiment + h64=$(nix hash to-base64 --type "$1" "$2") + [ "$h64" = "$4" ] + + sri=$(nix hash convert --hash-algo "$1" --to sri "$2") + [ "$sri" = "$1-$4" ] + sri=$(nix-hash --type "$1" --to-sri "$2") + [ "$sri" = "$1-$4" ] + sri=$(nix hash to-sri --type "$1" "$2") + [ "$sri" = "$1-$4" ] + h32=$(nix hash convert --hash-algo "$1" --to base32 "$2") + [ "$h32" = "$3" ] + h32=$(nix-hash --type "$1" --to-base32 "$2") + [ "$h32" = "$3" ] + h32=$(nix hash to-base32 --type "$1" "$2") + [ "$h32" = "$3" ] + h16=$(nix-hash --type "$1" --to-base16 "$h32") + [ "$h16" = "$2" ] + + h16=$(nix hash convert --hash-algo "$1" --to base16 "$h64") + [ "$h16" = "$2" ] + h16=$(nix hash to-base16 --type "$1" "$h64") + [ "$h16" = "$2" ] + h16=$(nix hash convert --to base16 "$sri") + [ "$h16" = "$2" ] + h16=$(nix hash to-base16 "$sri") + [ "$h16" = "$2" ] + + # + # Converting from SRI + # + + # Input hash algo auto-detected from SRI and output defaults to SRI as well. + sri=$(nix hash convert "$1-$4") + [ "$sri" = "$1-$4" ] + + sri=$(nix hash convert --from sri "$1-$4") + [ "$sri" = "$1-$4" ] + + sri=$(nix hash convert --to sri "$1-$4") + [ "$sri" = "$1-$4" ] + + sri=$(nix hash convert --from sri --to sri "$1-$4") + [ "$sri" = "$1-$4" ] + + sri=$(nix hash convert --to base64 "$1-$4") + [ "$sri" = "$4" ] + + # + # Auto-detecting the input from algo and length. + # + + sri=$(nix hash convert --hash-algo "$1" "$2") + [ "$sri" = "$1-$4" ] + sri=$(nix hash convert --hash-algo "$1" "$3") + [ "$sri" = "$1-$4" ] + sri=$(nix hash convert --hash-algo "$1" "$4") + [ "$sri" = "$1-$4" ] + + sri=$(nix hash convert --hash-algo "$1" "$2") + [ "$sri" = "$1-$4" ] + sri=$(nix hash convert --hash-algo "$1" "$3") + [ "$sri" = "$1-$4" ] + sri=$(nix hash convert --hash-algo "$1" "$4") + [ "$sri" = "$1-$4" ] + + # + # Asserting input format succeeds. + # + + sri=$(nix hash convert --hash-algo "$1" --from base16 "$2") + [ "$sri" = "$1-$4" ] + sri=$(nix hash convert --hash-algo "$1" --from nix32 "$3") + [ "$sri" = "$1-$4" ] + sri=$(nix hash convert --hash-algo "$1" --from base64 "$4") + [ "$sri" = "$1-$4" ] + + # + # Asserting input format fails. + # + + fail=$(nix hash convert --hash-algo "$1" --from nix32 "$2" 2>&1 || echo "exit: $?") + [[ "$fail" == *"error: input hash"*"exit: 1" ]] + fail=$(nix hash convert --hash-algo "$1" --from base16 "$3" 2>&1 || echo "exit: $?") + [[ "$fail" == *"error: input hash"*"exit: 1" ]] + fail=$(nix hash convert --hash-algo "$1" --from nix32 "$4" 2>&1 || echo "exit: $?") + [[ "$fail" == *"error: input hash"*"exit: 1" ]] + +} + +try3 sha1 "800d59cfcd3c05e900cb4e214be48f6b886a08df" "vw46m23bizj4n8afrc0fj19wrp7mj3c0" "gA1Zz808BekAy04hS+SPa4hqCN8=" +try3 sha256 "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=" +try3 sha512 "204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445" "12k9jiq29iyqm03swfsgiw5mlqs173qazm3n7daz43infy12pyrcdf30fkk3qwv4yl2ick8yipc2mqnlh48xsvvxl60lbx8vp38yji0" "IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ==" diff --git a/tests/functional/hash-path.sh b/tests/functional/hash-path.sh new file mode 100644 index 000000000..4ad9f8ff2 --- /dev/null +++ b/tests/functional/hash-path.sh @@ -0,0 +1,92 @@ +source common.sh + +try () { + printf "%s" "$2" > $TEST_ROOT/vector + hash="$(nix-hash --flat ${FORMAT+--$FORMAT} --type "$1" "$TEST_ROOT/vector")" + if ! (( "${NO_TEST_CLASSIC-}" )) && test "$hash" != "$3"; then + echo "try nix-hash: hash $1, expected $3, got $hash" + exit 1 + fi + hash="$(nix hash file ${FORMAT+--$FORMAT} --type "$1" "$TEST_ROOT/vector")" + if ! (( "${NO_TEST_NIX_COMMAND-}" )) && test "$hash" != "$3"; then + echo "try nix hash: hash $1, expected $3, got $hash" + exit 1 + fi + hash="$(nix hash path --mode flat ${FORMAT+--format $FORMAT} --algo "$1" "$TEST_ROOT/vector")" + if ! (( "${NO_TEST_NIX_COMMAND-}" )) && test "$hash" != "$3"; then + echo "try nix hash: hash $1, expected $3, got $hash" + exit 1 + fi +} + +FORMAT=base16 +try md5 "" "d41d8cd98f00b204e9800998ecf8427e" +try md5 "a" "0cc175b9c0f1b6a831c399e269772661" +try md5 "abc" "900150983cd24fb0d6963f7d28e17f72" +try md5 "message digest" "f96b697d7cb7938d525a2f31aaf161d0" +try md5 "abcdefghijklmnopqrstuvwxyz" "c3fcd3d76192e4007dfb496cca67e13b" +try md5 "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" "d174ab98d277d9f5a5611c2c9f419d9f" +try md5 "12345678901234567890123456789012345678901234567890123456789012345678901234567890" "57edf4a22be3c955ac49da2e2107b67a" + +try sha1 "" "da39a3ee5e6b4b0d3255bfef95601890afd80709" +try sha1 "abc" "a9993e364706816aba3e25717850c26c9cd0d89d" +try sha1 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "84983e441c3bd26ebaae4aa1f95129e5e54670f1" + +try sha256 "" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" +try sha256 "abc" "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" +try sha256 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1" + +try sha512 "" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" +try sha512 "abc" "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f" +try sha512 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445" +unset FORMAT + +FORMAT=base32 +try sha256 "abc" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" +unset FORMAT + +FORMAT=sri +try sha512 "" "sha512-z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" +try sha512 "abc" "sha512-3a81oZNherrMQXNJriBBMRLm+k6JqX6iCp7u5ktV05ohkpkqJ0/BqDa6PCOj/uu9RU1EI2Q86A4qmslPpUyknw==" +try sha512 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "sha512-IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ==" +try sha256 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "sha256-JI1qYdIGOLjlwCaTDD5gOaM85Flk/yFn9uzt1BnbBsE=" +unset FORMAT + +# nix-hash [--flat] defaults to the Base16 format +NO_TEST_NIX_COMMAND=1 try sha512 "abc" "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f" + +# nix hash [file|path] defaults to the SRI format +NO_TEST_CLASSIC=1 try sha512 "abc" "sha512-3a81oZNherrMQXNJriBBMRLm+k6JqX6iCp7u5ktV05ohkpkqJ0/BqDa6PCOj/uu9RU1EI2Q86A4qmslPpUyknw==" + +try2 () { + hash=$(nix-hash --type "$1" $TEST_ROOT/hash-path) + if test "$hash" != "$2"; then + echo "try nix-hash; hash $1, expected $2, got $hash" + exit 1 + fi + hash="$(nix hash path --mode nar --format base16 --algo "$1" "$TEST_ROOT/hash-path")" + if test "$hash" != "$2"; then + echo "try nix hash: hash $1, expected $2, got $hash" + exit 1 + fi +} + +rm -rf $TEST_ROOT/hash-path +mkdir $TEST_ROOT/hash-path +echo "Hello World" > $TEST_ROOT/hash-path/hello + +try2 md5 "ea9b55537dd4c7e104515b2ccfaf4100" + +# Execute bit matters. +chmod +x $TEST_ROOT/hash-path/hello +try2 md5 "20f3ffe011d4cfa7d72bfabef7882836" + +# Mtime and other bits don't. +touch -r . $TEST_ROOT/hash-path/hello +chmod 744 $TEST_ROOT/hash-path/hello +try2 md5 "20f3ffe011d4cfa7d72bfabef7882836" + +# File type (e.g., symlink) does. +rm $TEST_ROOT/hash-path/hello +ln -s x $TEST_ROOT/hash-path/hello +try2 md5 "f78b733a68f5edbdf9413899339eaa4a" diff --git a/tests/functional/hash.sh b/tests/functional/hash.sh deleted file mode 100644 index ff270076e..000000000 --- a/tests/functional/hash.sh +++ /dev/null @@ -1,186 +0,0 @@ -source common.sh - -try () { - printf "%s" "$2" > $TEST_ROOT/vector - hash="$(nix-hash --flat ${FORMAT_FLAG-} --type "$1" "$TEST_ROOT/vector")" - if ! (( "${NO_TEST_CLASSIC-}" )) && test "$hash" != "$3"; then - echo "try nix-hash: hash $1, expected $3, got $hash" - exit 1 - fi - hash="$(nix hash file ${FORMAT_FLAG-} --type "$1" "$TEST_ROOT/vector")" - if ! (( "${NO_TEST_NIX_COMMAND-}" )) && test "$hash" != "$3"; then - echo "try nix hash: hash $1, expected $3, got $hash" - exit 1 - fi -} - -FORMAT_FLAG=--base16 -try md5 "" "d41d8cd98f00b204e9800998ecf8427e" -try md5 "a" "0cc175b9c0f1b6a831c399e269772661" -try md5 "abc" "900150983cd24fb0d6963f7d28e17f72" -try md5 "message digest" "f96b697d7cb7938d525a2f31aaf161d0" -try md5 "abcdefghijklmnopqrstuvwxyz" "c3fcd3d76192e4007dfb496cca67e13b" -try md5 "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" "d174ab98d277d9f5a5611c2c9f419d9f" -try md5 "12345678901234567890123456789012345678901234567890123456789012345678901234567890" "57edf4a22be3c955ac49da2e2107b67a" - -try sha1 "" "da39a3ee5e6b4b0d3255bfef95601890afd80709" -try sha1 "abc" "a9993e364706816aba3e25717850c26c9cd0d89d" -try sha1 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "84983e441c3bd26ebaae4aa1f95129e5e54670f1" - -try sha256 "" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" -try sha256 "abc" "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" -try sha256 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1" - -try sha512 "" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" -try sha512 "abc" "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f" -try sha512 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445" -unset FORMAT_FLAG - -FORMAT_FLAG=--base32 -try sha256 "abc" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" -unset FORMAT_FLAG - -FORMAT_FLAG=--sri -try sha512 "" "sha512-z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" -try sha512 "abc" "sha512-3a81oZNherrMQXNJriBBMRLm+k6JqX6iCp7u5ktV05ohkpkqJ0/BqDa6PCOj/uu9RU1EI2Q86A4qmslPpUyknw==" -try sha512 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "sha512-IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ==" -try sha256 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "sha256-JI1qYdIGOLjlwCaTDD5gOaM85Flk/yFn9uzt1BnbBsE=" -unset FORMAT_FLAG - -# nix-hash [--flat] defaults to the Base16 format -NO_TEST_NIX_COMMAND=1 try sha512 "abc" "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f" - -# nix hash [file|path] defaults to the SRI format -NO_TEST_CLASSIC=1 try sha512 "abc" "sha512-3a81oZNherrMQXNJriBBMRLm+k6JqX6iCp7u5ktV05ohkpkqJ0/BqDa6PCOj/uu9RU1EI2Q86A4qmslPpUyknw==" - -try2 () { - hash=$(nix-hash --type "$1" $TEST_ROOT/hash-path) - if test "$hash" != "$2"; then - echo "hash $1, expected $2, got $hash" - exit 1 - fi -} - -rm -rf $TEST_ROOT/hash-path -mkdir $TEST_ROOT/hash-path -echo "Hello World" > $TEST_ROOT/hash-path/hello - -try2 md5 "ea9b55537dd4c7e104515b2ccfaf4100" - -# Execute bit matters. -chmod +x $TEST_ROOT/hash-path/hello -try2 md5 "20f3ffe011d4cfa7d72bfabef7882836" - -# Mtime and other bits don't. -touch -r . $TEST_ROOT/hash-path/hello -chmod 744 $TEST_ROOT/hash-path/hello -try2 md5 "20f3ffe011d4cfa7d72bfabef7882836" - -# File type (e.g., symlink) does. -rm $TEST_ROOT/hash-path/hello -ln -s x $TEST_ROOT/hash-path/hello -try2 md5 "f78b733a68f5edbdf9413899339eaa4a" - -# Conversion with `nix hash` `nix-hash` and `nix hash convert` -try3() { - # $1 = hash algo - # $2 = expected hash in base16 - # $3 = expected hash in base32 - # $4 = expected hash in base64 - h64=$(nix hash convert --hash-algo "$1" --to base64 "$2") - [ "$h64" = "$4" ] - h64=$(nix-hash --type "$1" --to-base64 "$2") - [ "$h64" = "$4" ] - # Deprecated experiment - h64=$(nix hash to-base64 --type "$1" "$2") - [ "$h64" = "$4" ] - - sri=$(nix hash convert --hash-algo "$1" --to sri "$2") - [ "$sri" = "$1-$4" ] - sri=$(nix-hash --type "$1" --to-sri "$2") - [ "$sri" = "$1-$4" ] - sri=$(nix hash to-sri --type "$1" "$2") - [ "$sri" = "$1-$4" ] - h32=$(nix hash convert --hash-algo "$1" --to base32 "$2") - [ "$h32" = "$3" ] - h32=$(nix-hash --type "$1" --to-base32 "$2") - [ "$h32" = "$3" ] - h32=$(nix hash to-base32 --type "$1" "$2") - [ "$h32" = "$3" ] - h16=$(nix-hash --type "$1" --to-base16 "$h32") - [ "$h16" = "$2" ] - - h16=$(nix hash convert --hash-algo "$1" --to base16 "$h64") - [ "$h16" = "$2" ] - h16=$(nix hash to-base16 --type "$1" "$h64") - [ "$h16" = "$2" ] - h16=$(nix hash convert --to base16 "$sri") - [ "$h16" = "$2" ] - h16=$(nix hash to-base16 "$sri") - [ "$h16" = "$2" ] - - # - # Converting from SRI - # - - # Input hash algo auto-detected from SRI and output defaults to SRI as well. - sri=$(nix hash convert "$1-$4") - [ "$sri" = "$1-$4" ] - - sri=$(nix hash convert --from sri "$1-$4") - [ "$sri" = "$1-$4" ] - - sri=$(nix hash convert --to sri "$1-$4") - [ "$sri" = "$1-$4" ] - - sri=$(nix hash convert --from sri --to sri "$1-$4") - [ "$sri" = "$1-$4" ] - - sri=$(nix hash convert --to base64 "$1-$4") - [ "$sri" = "$4" ] - - # - # Auto-detecting the input from algo and length. - # - - sri=$(nix hash convert --hash-algo "$1" "$2") - [ "$sri" = "$1-$4" ] - sri=$(nix hash convert --hash-algo "$1" "$3") - [ "$sri" = "$1-$4" ] - sri=$(nix hash convert --hash-algo "$1" "$4") - [ "$sri" = "$1-$4" ] - - sri=$(nix hash convert --hash-algo "$1" "$2") - [ "$sri" = "$1-$4" ] - sri=$(nix hash convert --hash-algo "$1" "$3") - [ "$sri" = "$1-$4" ] - sri=$(nix hash convert --hash-algo "$1" "$4") - [ "$sri" = "$1-$4" ] - - # - # Asserting input format succeeds. - # - - sri=$(nix hash convert --hash-algo "$1" --from base16 "$2") - [ "$sri" = "$1-$4" ] - sri=$(nix hash convert --hash-algo "$1" --from nix32 "$3") - [ "$sri" = "$1-$4" ] - sri=$(nix hash convert --hash-algo "$1" --from base64 "$4") - [ "$sri" = "$1-$4" ] - - # - # Asserting input format fails. - # - - fail=$(nix hash convert --hash-algo "$1" --from nix32 "$2" 2>&1 || echo "exit: $?") - [[ "$fail" == *"error: input hash"*"exit: 1" ]] - fail=$(nix hash convert --hash-algo "$1" --from base16 "$3" 2>&1 || echo "exit: $?") - [[ "$fail" == *"error: input hash"*"exit: 1" ]] - fail=$(nix hash convert --hash-algo "$1" --from nix32 "$4" 2>&1 || echo "exit: $?") - [[ "$fail" == *"error: input hash"*"exit: 1" ]] - -} - -try3 sha1 "800d59cfcd3c05e900cb4e214be48f6b886a08df" "vw46m23bizj4n8afrc0fj19wrp7mj3c0" "gA1Zz808BekAy04hS+SPa4hqCN8=" -try3 sha256 "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=" -try3 sha512 "204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445" "12k9jiq29iyqm03swfsgiw5mlqs173qazm3n7daz43infy12pyrcdf30fkk3qwv4yl2ick8yipc2mqnlh48xsvvxl60lbx8vp38yji0" "IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ==" diff --git a/tests/functional/impure-derivations.sh b/tests/functional/impure-derivations.sh index 39d053a04..54ed6f5dd 100644 --- a/tests/functional/impure-derivations.sh +++ b/tests/functional/impure-derivations.sh @@ -63,3 +63,7 @@ path5=$(nix build -L --no-link --json --file ./impure-derivations.nix contentAdd path6=$(nix build -L --no-link --json --file ./impure-derivations.nix inputAddressedAfterCA | jq -r .[].outputs.out) [[ $(< $path6) = X ]] [[ $(< $TEST_ROOT/counter) = 5 ]] + +# Test nix/fetchurl.nix. +path7=$(nix build -L --no-link --print-out-paths --expr "import { impure = true; url = file://$PWD/impure-derivations.sh; }") +cmp $path7 $PWD/impure-derivations.sh diff --git a/tests/functional/impure-env.sh b/tests/functional/impure-env.sh index d9e4a34a2..cfea4cae9 100644 --- a/tests/functional/impure-env.sh +++ b/tests/functional/impure-env.sh @@ -1,7 +1,7 @@ source common.sh # Needs the config option 'impure-env' to work -requireDaemonNewerThan "2.18.0pre20230816" +requireDaemonNewerThan "2.19.0" enableFeatures "configurable-impure-env" restartDaemon diff --git a/tests/functional/lang/eval-fail-attr-name-type.err.exp b/tests/functional/lang/eval-fail-attr-name-type.err.exp index c8d56ba7d..6848a35ed 100644 --- a/tests/functional/lang/eval-fail-attr-name-type.err.exp +++ b/tests/functional/lang/eval-fail-attr-name-type.err.exp @@ -14,3 +14,8 @@ error: 8| error: expected a string but found an integer: 1 + at /pwd/lang/eval-fail-attr-name-type.nix:7:17: + 6| in + 7| attrs.puppy.${key} + | ^ + 8| diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp index 5119238d7..6f907106b 100644 --- a/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp @@ -6,4 +6,4 @@ error: | ^ 10| - error: cannot coerce a set to a string: { a = { a = { a = { a = "ha"; b = "ha"; c = "ha"; d = "ha"; e = "ha"; f = "ha"; g = "ha"; h = "ha"; j = "ha"; }; «4294967295 attributes elided»}; «4294967294 attributes elided»}; «4294967293 attributes elided»} + error: cannot coerce a set to a string: { a = { a = { a = { a = "ha"; b = "ha"; c = "ha"; d = "ha"; e = "ha"; f = "ha"; g = "ha"; h = "ha"; j = "ha"; }; «4294967295 attributes elided» }; «4294967294 attributes elided» }; «4294967293 attributes elided» } diff --git a/tests/functional/lang/eval-fail-duplicate-traces.err.exp b/tests/functional/lang/eval-fail-duplicate-traces.err.exp index 32ad9b376..cedaebd3b 100644 --- a/tests/functional/lang/eval-fail-duplicate-traces.err.exp +++ b/tests/functional/lang/eval-fail-duplicate-traces.err.exp @@ -41,4 +41,11 @@ error: | ^ 5| if n > 0 + … while calling the 'throw' builtin + at /pwd/lang/eval-fail-duplicate-traces.nix:7:10: + 6| then throwAfter (n - 1) + 7| else throw "Uh oh!"; + | ^ + 8| in + error: Uh oh! diff --git a/tests/functional/lang/eval-fail-foldlStrict-strict-op-application.err.exp b/tests/functional/lang/eval-fail-foldlStrict-strict-op-application.err.exp index 7cb08af8a..4903bc82d 100644 --- a/tests/functional/lang/eval-fail-foldlStrict-strict-op-application.err.exp +++ b/tests/functional/lang/eval-fail-foldlStrict-strict-op-application.err.exp @@ -27,4 +27,11 @@ error: | ^ 6| + … while calling the 'throw' builtin + at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:5:9: + 4| null + 5| [ (_: throw "Not the final value, but is still forced!") (_: 23) ] + | ^ + 6| + error: Not the final value, but is still forced! diff --git a/tests/functional/lang/eval-fail-fromTOML-timestamps.err.exp b/tests/functional/lang/eval-fail-fromTOML-timestamps.err.exp index 73f9df8cc..9bbb251e1 100644 --- a/tests/functional/lang/eval-fail-fromTOML-timestamps.err.exp +++ b/tests/functional/lang/eval-fail-fromTOML-timestamps.err.exp @@ -5,4 +5,4 @@ error: | ^ 2| key = "value" - error: while parsing a TOML string: Dates and times are not supported + error: while parsing TOML: Dates and times are not supported diff --git a/tests/functional/lang/eval-fail-mutual-recursion.err.exp b/tests/functional/lang/eval-fail-mutual-recursion.err.exp index dc2e11766..c034afcd5 100644 --- a/tests/functional/lang/eval-fail-mutual-recursion.err.exp +++ b/tests/functional/lang/eval-fail-mutual-recursion.err.exp @@ -54,4 +54,11 @@ error: (21 duplicate frames omitted) + … while calling the 'throw' builtin + at /pwd/lang/eval-fail-mutual-recursion.nix:34:10: + 33| then throwAfterB true 10 + 34| else throw "Uh oh!"; + | ^ + 35| in + error: Uh oh! diff --git a/tests/functional/lang/eval-fail-toJSON.err.exp b/tests/functional/lang/eval-fail-toJSON.err.exp index 4f6003437..ad267711b 100644 --- a/tests/functional/lang/eval-fail-toJSON.err.exp +++ b/tests/functional/lang/eval-fail-toJSON.err.exp @@ -20,6 +20,11 @@ error: 3| true … while evaluating list element at index 3 + at /pwd/lang/eval-fail-toJSON.nix:2:3: + 1| builtins.toJSON { + 2| a.b = [ + | ^ + 3| true … while evaluating attribute 'c' at /pwd/lang/eval-fail-toJSON.nix:7:7: diff --git a/tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp b/tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp index 94784c651..4326c9650 100644 --- a/tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp +++ b/tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp @@ -7,3 +7,8 @@ error: 6| error: expected a string but found a set: { } + at /pwd/lang/eval-fail-using-set-as-attr-name.nix:5:10: + 4| in + 5| attr.${key} + | ^ + 6| diff --git a/tests/functional/lang/eval-okay-inherit-from.err.exp b/tests/functional/lang/eval-okay-inherit-from.err.exp new file mode 100644 index 000000000..3227501f2 --- /dev/null +++ b/tests/functional/lang/eval-okay-inherit-from.err.exp @@ -0,0 +1 @@ +trace: used diff --git a/tests/functional/lang/eval-okay-inherit-from.exp b/tests/functional/lang/eval-okay-inherit-from.exp new file mode 100644 index 000000000..024daff6b --- /dev/null +++ b/tests/functional/lang/eval-okay-inherit-from.exp @@ -0,0 +1 @@ +[ 1 2 { __overrides = { y = { d = [ ]; }; }; c = [ ]; d = 4; x = { c = [ ]; }; y = «repeated»; } { inner = { c = 3; d = 4; }; } ] diff --git a/tests/functional/lang/eval-okay-inherit-from.nix b/tests/functional/lang/eval-okay-inherit-from.nix new file mode 100644 index 000000000..b72a1c639 --- /dev/null +++ b/tests/functional/lang/eval-okay-inherit-from.nix @@ -0,0 +1,16 @@ +let + inherit (builtins.trace "used" { a = 1; b = 2; }) a b; + x.c = 3; + y.d = 4; + + merged = { + inner = { + inherit (y) d; + }; + + inner = { + inherit (x) c; + }; + }; +in + [ a b rec { x.c = []; inherit (x) c; inherit (y) d; __overrides.y.d = []; } merged ] diff --git a/tests/functional/lang/eval-okay-pathexists.nix b/tests/functional/lang/eval-okay-pathexists.nix index 31697f66a..022b22fea 100644 --- a/tests/functional/lang/eval-okay-pathexists.nix +++ b/tests/functional/lang/eval-okay-pathexists.nix @@ -29,3 +29,6 @@ builtins.pathExists (./lib.nix) && builtins.pathExists (builtins.toPath { outPath = builtins.toString ./lib.nix; }) && builtins.pathExists ./lib.nix && !builtins.pathExists ./bla.nix +&& builtins.pathExists ./symlink-resolution/foo/overlays/overlay.nix +&& builtins.pathExists ./symlink-resolution/broken +&& builtins.pathExists (builtins.toString ./symlink-resolution/foo/overlays + "/.") diff --git a/tests/functional/lang/parse-okay-inherits.exp b/tests/functional/lang/parse-okay-inherits.exp new file mode 100644 index 000000000..1355527e6 --- /dev/null +++ b/tests/functional/lang/parse-okay-inherits.exp @@ -0,0 +1 @@ +(let b = 2; c = { }; in { inherit b; inherit (c) d e; a = 1; f = 3; }) diff --git a/tests/functional/lang/parse-okay-inherits.nix b/tests/functional/lang/parse-okay-inherits.nix new file mode 100644 index 000000000..10596c8ad --- /dev/null +++ b/tests/functional/lang/parse-okay-inherits.nix @@ -0,0 +1,9 @@ +let + c = {}; + b = 2; +in { + a = 1; + inherit b; + inherit (c) d e; + f = 3; +} diff --git a/tests/functional/lang/parse-okay-subversion.exp b/tests/functional/lang/parse-okay-subversion.exp index 4168ee8bf..2303932c4 100644 --- a/tests/functional/lang/parse-okay-subversion.exp +++ b/tests/functional/lang/parse-okay-subversion.exp @@ -1 +1 @@ -({ fetchurl, localServer ? false, httpServer ? false, sslSupport ? false, pythonBindings ? false, javaSwigBindings ? false, javahlBindings ? false, stdenv, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null }: assert (expat != null); assert (localServer -> (db4 != null)); assert (httpServer -> ((httpd != null) && ((httpd).expat == expat))); assert (sslSupport -> ((openssl != null) && (httpServer -> ((httpd).openssl == openssl)))); assert (pythonBindings -> ((swig != null) && (swig).pythonSupport)); assert (javaSwigBindings -> ((swig != null) && (swig).javaSupport)); assert (javahlBindings -> (j2sdk != null)); ((stdenv).mkDerivation { builder = /foo/bar; db4 = (if localServer then db4 else null); inherit expat ; inherit httpServer ; httpd = (if httpServer then httpd else null); j2sdk = (if javaSwigBindings then (swig).j2sdk else (if javahlBindings then j2sdk else null)); inherit javaSwigBindings ; inherit javahlBindings ; inherit localServer ; name = "subversion-1.1.1"; openssl = (if sslSupport then openssl else null); patches = (if javahlBindings then [ (/javahl.patch) ] else [ ]); python = (if pythonBindings then (swig).python else null); inherit pythonBindings ; src = (fetchurl { md5 = "a180c3fe91680389c210c99def54d9e0"; url = "http://subversion.tigris.org/tarballs/subversion-1.1.1.tar.bz2"; }); inherit sslSupport ; swig = (if (pythonBindings || javaSwigBindings) then swig else null); })) +({ fetchurl, localServer ? false, httpServer ? false, sslSupport ? false, pythonBindings ? false, javaSwigBindings ? false, javahlBindings ? false, stdenv, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null }: assert (expat != null); assert (localServer -> (db4 != null)); assert (httpServer -> ((httpd != null) && ((httpd).expat == expat))); assert (sslSupport -> ((openssl != null) && (httpServer -> ((httpd).openssl == openssl)))); assert (pythonBindings -> ((swig != null) && (swig).pythonSupport)); assert (javaSwigBindings -> ((swig != null) && (swig).javaSupport)); assert (javahlBindings -> (j2sdk != null)); ((stdenv).mkDerivation { inherit expat httpServer javaSwigBindings javahlBindings localServer pythonBindings sslSupport; builder = /foo/bar; db4 = (if localServer then db4 else null); httpd = (if httpServer then httpd else null); j2sdk = (if javaSwigBindings then (swig).j2sdk else (if javahlBindings then j2sdk else null)); name = "subversion-1.1.1"; openssl = (if sslSupport then openssl else null); patches = (if javahlBindings then [ (/javahl.patch) ] else [ ]); python = (if pythonBindings then (swig).python else null); src = (fetchurl { md5 = "a180c3fe91680389c210c99def54d9e0"; url = "http://subversion.tigris.org/tarballs/subversion-1.1.1.tar.bz2"; }); swig = (if (pythonBindings || javaSwigBindings) then swig else null); })) diff --git a/tests/functional/lang/symlink-resolution/broken b/tests/functional/lang/symlink-resolution/broken new file mode 120000 index 000000000..e07da690b --- /dev/null +++ b/tests/functional/lang/symlink-resolution/broken @@ -0,0 +1 @@ +nonexistent \ No newline at end of file diff --git a/tests/functional/local.mk b/tests/functional/local.mk index 888c7e18a..18eb887cd 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -47,7 +47,8 @@ nix_tests = \ optimise-store.sh \ substitute-with-invalid-ca.sh \ signing.sh \ - hash.sh \ + hash-convert.sh \ + hash-path.sh \ gc-non-blocking.sh \ check.sh \ nix-shell.sh \ @@ -127,7 +128,8 @@ nix_tests = \ toString-path.sh \ read-only-store.sh \ nested-sandboxing.sh \ - impure-env.sh + impure-env.sh \ + debugger.sh ifeq ($(HAVE_LIBCPUID), 1) nix_tests += compute-levels.sh diff --git a/tests/functional/nix-profile.sh b/tests/functional/nix-profile.sh index 35a62fbe2..ee93251e9 100644 --- a/tests/functional/nix-profile.sh +++ b/tests/functional/nix-profile.sh @@ -64,6 +64,9 @@ nix profile install $flake1Dir [[ $($TEST_HOME/.local/state/nix/profile/bin/hello) = "Hello World" ]] unset NIX_CONFIG +# Test conflicting package install. +nix profile install $flake1Dir 2>&1 | grep "warning: 'flake1' is already installed" + # Test upgrading a package. printf NixOS > $flake1Dir/who printf 2.0 > $flake1Dir/version @@ -166,7 +169,7 @@ error: An existing package already provides the following file: To remove the existing package: - nix profile remove path:${flake1Dir}#packages.${system}.default + nix profile remove flake1 The new package can also be installed next to the existing one by assigning a different priority. The conflicting packages have a priority of 5. diff --git a/tests/functional/nix-shell.sh b/tests/functional/nix-shell.sh index 13403fadb..04c83138e 100644 --- a/tests/functional/nix-shell.sh +++ b/tests/functional/nix-shell.sh @@ -118,10 +118,10 @@ diff $TEST_ROOT/dev-env{,2}.json # Ensure `nix print-dev-env --json` contains variable assignments. [[ $(jq -r .variables.arr1.value[2] $TEST_ROOT/dev-env.json) = '3 4' ]] -# Run tests involving `source <(nix print-dev-inv)` in subshells to avoid modifying the current +# Run tests involving `source <(nix print-dev-env)` in subshells to avoid modifying the current # environment. -set +u # FIXME: Make print-dev-env `set -u` compliant (issue #7951) +set -u # Ensure `source <(nix print-dev-env)` modifies the environment. ( diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index 1b779c1f5..4938c2267 100644 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -146,29 +146,86 @@ echo "$replResult" | grepQuiet -s afterChange # Normal output should print attributes in lexicographical order non-recursively testReplResponseNoRegex ' { a = { b = 2; }; l = [ 1 2 3 ]; s = "string"; n = 1234; x = rec { y = { z = { inherit y; }; }; }; } -' '{ a = { ... }; l = [ ... ]; n = 1234; s = "string"; x = { ... }; }' +' \ +'{ + a = { ... }; + l = [ ... ]; + n = 1234; + s = "string"; + x = { ... }; +} +' # Same for lists, but order is preserved testReplResponseNoRegex ' [ 42 1 "thingy" ({ a = 1; }) ([ 1 2 3 ]) ] -' '[ 42 1 "thingy" { ... } [ ... ] ]' +' \ +'[ + 42 + 1 + "thingy" + { ... } + [ ... ] +] +' # Same for let expressions testReplResponseNoRegex ' let x = { y = { a = 1; }; inherit x; }; in x -' '{ x = { ... }; y = { ... }; }' +' \ +'{ + x = { ... }; + y = { ... }; +} +' # The :p command should recursively print sets, but prevent infinite recursion testReplResponseNoRegex ' :p { a = { b = 2; }; s = "string"; n = 1234; x = rec { y = { z = { inherit y; }; }; }; } -' '{ a = { b = 2; }; n = 1234; s = "string"; x = { y = { z = { y = «repeated»; }; }; }; }' +' \ +'{ + a = { b = 2; }; + n = 1234; + s = "string"; + x = { + y = { + z = { + y = «repeated»; + }; + }; + }; +} +' # Same for lists testReplResponseNoRegex ' :p [ 42 1 "thingy" (rec { a = 1; b = { inherit a; inherit b; }; }) ([ 1 2 3 ]) ] -' '[ 42 1 "thingy" { a = 1; b = { a = 1; b = «repeated»; }; } [ 1 2 3 ] ]' +' \ +'[ + 42 + 1 + "thingy" + { + a = 1; + b = { + a = 1; + b = «repeated»; + }; + } + [ + 1 + 2 + 3 + ] +] +' # Same for let expressions testReplResponseNoRegex ' :p let x = { y = { a = 1; }; inherit x; }; in x -' '{ x = { x = «repeated»; y = { a = 1; }; }; y = «repeated»; }' +' \ +'{ + x = «repeated»; + y = { a = 1 }; +} +' diff --git a/tests/functional/tarball.sh b/tests/functional/tarball.sh index e59ee400e..062f27ad6 100644 --- a/tests/functional/tarball.sh +++ b/tests/functional/tarball.sh @@ -42,11 +42,11 @@ test_tarball() { nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" >&2 nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" 2>&1 | grep 'true' - nix-instantiate --eval -E '1 + 2' -I fnord=file://no-such-tarball.tar$ext - nix-instantiate --eval -E 'with ; 1 + 2' -I fnord=file://no-such-tarball$ext - (! nix-instantiate --eval -E ' 1' -I fnord=file://no-such-tarball$ext) + nix-instantiate --eval -E '1 + 2' -I fnord=file:///no-such-tarball.tar$ext + nix-instantiate --eval -E 'with ; 1 + 2' -I fnord=file:///no-such-tarball$ext + (! nix-instantiate --eval -E ' 1' -I fnord=file:///no-such-tarball$ext) - nix-instantiate --eval -E '' -I fnord=file://no-such-tarball$ext -I fnord=. + nix-instantiate --eval -E '' -I fnord=file:///no-such-tarball$ext -I fnord=. # Ensure that the `name` attribute isn’t accepted as that would mess # with the content-addressing @@ -57,8 +57,3 @@ test_tarball() { test_tarball '' cat test_tarball .xz xz test_tarball .gz gzip - -rm -rf $TEST_ROOT/tmp -mkdir -p $TEST_ROOT/tmp -(! TMPDIR=$TEST_ROOT/tmp XDG_RUNTIME_DIR=$TEST_ROOT/tmp nix-env -f file://$(pwd)/bad.tar.xz -qa --out-path) -(! [ -e $TEST_ROOT/tmp/bad ]) diff --git a/tests/unit/libexpr/error_traces.cc b/tests/unit/libexpr/error_traces.cc index 5fca79304..7b32b320b 100644 --- a/tests/unit/libexpr/error_traces.cc +++ b/tests/unit/libexpr/error_traces.cc @@ -12,33 +12,33 @@ namespace nix { TEST_F(ErrorTraceTest, TraceBuilder) { ASSERT_THROW( - state.error("Not much").debugThrow(), + state.error("puppy").debugThrow(), EvalError ); ASSERT_THROW( - state.error("Not much").withTrace(noPos, "No more").debugThrow(), + state.error("puppy").withTrace(noPos, "doggy").debugThrow(), EvalError ); ASSERT_THROW( try { try { - state.error("Not much").withTrace(noPos, "No more").debugThrow(); + state.error("puppy").withTrace(noPos, "doggy").debugThrow(); } catch (Error & e) { - e.addTrace(state.positions[noPos], "Something", ""); + e.addTrace(state.positions[noPos], "beans"); throw; } } catch (BaseError & e) { ASSERT_EQ(PrintToString(e.info().msg), - PrintToString(hintfmt("Not much"))); + PrintToString(HintFmt("puppy"))); auto trace = e.info().traces.rbegin(); ASSERT_EQ(e.info().traces.size(), 2); ASSERT_EQ(PrintToString(trace->hint), - PrintToString(hintfmt("No more"))); + PrintToString(HintFmt("doggy"))); trace++; ASSERT_EQ(PrintToString(trace->hint), - PrintToString(hintfmt("Something"))); + PrintToString(HintFmt("beans"))); throw; } , EvalError @@ -47,12 +47,12 @@ namespace nix { TEST_F(ErrorTraceTest, NestedThrows) { try { - state.error("Not much").withTrace(noPos, "No more").debugThrow(); + state.error("puppy").withTrace(noPos, "doggy").debugThrow(); } catch (BaseError & e) { try { - state.error("Not much more").debugThrow(); + state.error("beans").debugThrow(); } catch (Error & e2) { - e.addTrace(state.positions[noPos], "Something", ""); + e.addTrace(state.positions[noPos], "beans2"); //e2.addTrace(state.positions[noPos], "Something", ""); ASSERT_TRUE(e.info().traces.size() == 2); ASSERT_TRUE(e2.info().traces.size() == 0); @@ -74,7 +74,7 @@ namespace nix { ASSERT_EQ(e.info().traces.size(), 1) << "while testing " args << std::endl << e.what(); \ auto trace = e.info().traces.rbegin(); \ ASSERT_EQ(PrintToString(trace->hint), \ - PrintToString(hintfmt("while calling the '%s' builtin", name))); \ + PrintToString(HintFmt("while calling the '%s' builtin", name))); \ throw; \ } \ , type \ @@ -96,7 +96,7 @@ namespace nix { PrintToString(context)); \ ++trace; \ ASSERT_EQ(PrintToString(trace->hint), \ - PrintToString(hintfmt("while calling the '%s' builtin", name))); \ + PrintToString(HintFmt("while calling the '%s' builtin", name))); \ throw; \ } \ , type \ @@ -105,48 +105,48 @@ namespace nix { TEST_F(ErrorTraceTest, genericClosure) { ASSERT_TRACE2("genericClosure 1", TypeError, - hintfmt("expected a set but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating the first argument passed to builtins.genericClosure")); + HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.genericClosure")); ASSERT_TRACE2("genericClosure {}", TypeError, - hintfmt("attribute '%s' missing", "startSet"), - hintfmt("in the attrset passed as argument to builtins.genericClosure")); + HintFmt("attribute '%s' missing", "startSet"), + HintFmt("in the attrset passed as argument to builtins.genericClosure")); ASSERT_TRACE2("genericClosure { startSet = 1; }", TypeError, - hintfmt("expected a list but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating the 'startSet' attribute passed as argument to builtins.genericClosure")); + HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the 'startSet' attribute passed as argument to builtins.genericClosure")); ASSERT_TRACE2("genericClosure { startSet = [{ key = 1;}]; operator = true; }", TypeError, - hintfmt("expected a function but found %s: %s", "a Boolean", ANSI_CYAN "true" ANSI_NORMAL), - hintfmt("while evaluating the 'operator' attribute passed as argument to builtins.genericClosure")); + HintFmt("expected a function but found %s: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), + HintFmt("while evaluating the 'operator' attribute passed as argument to builtins.genericClosure")); ASSERT_TRACE2("genericClosure { startSet = [{ key = 1;}]; operator = item: true; }", TypeError, - hintfmt("expected a list but found %s: %s", "a Boolean", ANSI_CYAN "true" ANSI_NORMAL), - hintfmt("while evaluating the return value of the `operator` passed to builtins.genericClosure")); + HintFmt("expected a list but found %s: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), + HintFmt("while evaluating the return value of the `operator` passed to builtins.genericClosure")); ASSERT_TRACE2("genericClosure { startSet = [{ key = 1;}]; operator = item: [ true ]; }", TypeError, - hintfmt("expected a set but found %s: %s", "a Boolean", ANSI_CYAN "true" ANSI_NORMAL), - hintfmt("while evaluating one of the elements generated by (or initially passed to) builtins.genericClosure")); + HintFmt("expected a set but found %s: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), + HintFmt("while evaluating one of the elements generated by (or initially passed to) builtins.genericClosure")); ASSERT_TRACE2("genericClosure { startSet = [{ key = 1;}]; operator = item: [ {} ]; }", TypeError, - hintfmt("attribute '%s' missing", "key"), - hintfmt("in one of the attrsets generated by (or initially passed to) builtins.genericClosure")); + HintFmt("attribute '%s' missing", "key"), + HintFmt("in one of the attrsets generated by (or initially passed to) builtins.genericClosure")); ASSERT_TRACE2("genericClosure { startSet = [{ key = 1;}]; operator = item: [{ key = ''a''; }]; }", EvalError, - hintfmt("cannot compare %s with %s", "a string", "an integer"), - hintfmt("while comparing the `key` attributes of two genericClosure elements")); + HintFmt("cannot compare %s with %s", "a string", "an integer"), + HintFmt("while comparing the `key` attributes of two genericClosure elements")); ASSERT_TRACE2("genericClosure { startSet = [ true ]; operator = item: [{ key = ''a''; }]; }", TypeError, - hintfmt("expected a set but found %s: %s", "a Boolean", ANSI_CYAN "true" ANSI_NORMAL), - hintfmt("while evaluating one of the elements generated by (or initially passed to) builtins.genericClosure")); + HintFmt("expected a set but found %s: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), + HintFmt("while evaluating one of the elements generated by (or initially passed to) builtins.genericClosure")); } @@ -154,32 +154,32 @@ namespace nix { TEST_F(ErrorTraceTest, replaceStrings) { ASSERT_TRACE2("replaceStrings 0 0 {}", TypeError, - hintfmt("expected a list but found %s: %s", "an integer", ANSI_CYAN "0" ANSI_NORMAL), - hintfmt("while evaluating the first argument passed to builtins.replaceStrings")); + HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "0" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.replaceStrings")); ASSERT_TRACE2("replaceStrings [] 0 {}", TypeError, - hintfmt("expected a list but found %s: %s", "an integer", ANSI_CYAN "0" ANSI_NORMAL), - hintfmt("while evaluating the second argument passed to builtins.replaceStrings")); + HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "0" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.replaceStrings")); ASSERT_TRACE1("replaceStrings [ 0 ] [] {}", EvalError, - hintfmt("'from' and 'to' arguments passed to builtins.replaceStrings have different lengths")); + HintFmt("'from' and 'to' arguments passed to builtins.replaceStrings have different lengths")); ASSERT_TRACE2("replaceStrings [ 1 ] [ \"new\" ] {}", TypeError, - hintfmt("expected a string but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating one of the strings to replace passed to builtins.replaceStrings")); + HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating one of the strings to replace passed to builtins.replaceStrings")); ASSERT_TRACE2("replaceStrings [ \"oo\" ] [ true ] \"foo\"", TypeError, - hintfmt("expected a string but found %s: %s", "a Boolean", ANSI_CYAN "true" ANSI_NORMAL), - hintfmt("while evaluating one of the replacement strings passed to builtins.replaceStrings")); + HintFmt("expected a string but found %s: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), + HintFmt("while evaluating one of the replacement strings passed to builtins.replaceStrings")); ASSERT_TRACE2("replaceStrings [ \"old\" ] [ \"new\" ] {}", TypeError, - hintfmt("expected a string but found %s: %s", "a set", "{ }"), - hintfmt("while evaluating the third argument passed to builtins.replaceStrings")); + HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), + HintFmt("while evaluating the third argument passed to builtins.replaceStrings")); } @@ -243,8 +243,8 @@ namespace nix { TEST_F(ErrorTraceTest, ceil) { ASSERT_TRACE2("ceil \"foo\"", TypeError, - hintfmt("expected a float but found %s: %s", "a string", ANSI_MAGENTA "\"foo\"" ANSI_NORMAL), - hintfmt("while evaluating the first argument passed to builtins.ceil")); + HintFmt("expected a float but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.ceil")); } @@ -252,8 +252,8 @@ namespace nix { TEST_F(ErrorTraceTest, floor) { ASSERT_TRACE2("floor \"foo\"", TypeError, - hintfmt("expected a float but found %s: %s", "a string", ANSI_MAGENTA "\"foo\"" ANSI_NORMAL), - hintfmt("while evaluating the first argument passed to builtins.floor")); + HintFmt("expected a float but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.floor")); } @@ -265,8 +265,8 @@ namespace nix { TEST_F(ErrorTraceTest, getEnv) { ASSERT_TRACE2("getEnv [ ]", TypeError, - hintfmt("expected a string but found %s: %s", "a list", "[ ]"), - hintfmt("while evaluating the first argument passed to builtins.getEnv")); + HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the first argument passed to builtins.getEnv")); } @@ -286,8 +286,8 @@ namespace nix { TEST_F(ErrorTraceTest, placeholder) { ASSERT_TRACE2("placeholder []", TypeError, - hintfmt("expected a string but found %s: %s", "a list", "[ ]"), - hintfmt("while evaluating the first argument passed to builtins.placeholder")); + HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the first argument passed to builtins.placeholder")); } @@ -295,13 +295,13 @@ namespace nix { TEST_F(ErrorTraceTest, toPath) { ASSERT_TRACE2("toPath []", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a list", "[ ]"), - hintfmt("while evaluating the first argument passed to builtins.toPath")); + HintFmt("cannot coerce %s to a string: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the first argument passed to builtins.toPath")); ASSERT_TRACE2("toPath \"foo\"", EvalError, - hintfmt("string '%s' doesn't represent an absolute path", "foo"), - hintfmt("while evaluating the first argument passed to builtins.toPath")); + HintFmt("string '%s' doesn't represent an absolute path", "foo"), + HintFmt("while evaluating the first argument passed to builtins.toPath")); } @@ -309,8 +309,8 @@ namespace nix { TEST_F(ErrorTraceTest, storePath) { ASSERT_TRACE2("storePath true", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a Boolean", ANSI_CYAN "true" ANSI_NORMAL), - hintfmt("while evaluating the first argument passed to 'builtins.storePath'")); + HintFmt("cannot coerce %s to a string: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to 'builtins.storePath'")); } @@ -318,13 +318,13 @@ namespace nix { TEST_F(ErrorTraceTest, pathExists) { ASSERT_TRACE2("pathExists []", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a list", "[ ]"), - hintfmt("while realising the context of a path")); + HintFmt("cannot coerce %s to a string: %s", "a list", Uncolored("[ ]")), + HintFmt("while realising the context of a path")); ASSERT_TRACE2("pathExists \"zorglub\"", EvalError, - hintfmt("string '%s' doesn't represent an absolute path", "zorglub"), - hintfmt("while realising the context of a path")); + HintFmt("string '%s' doesn't represent an absolute path", "zorglub"), + HintFmt("while realising the context of a path")); } @@ -332,8 +332,8 @@ namespace nix { TEST_F(ErrorTraceTest, baseNameOf) { ASSERT_TRACE2("baseNameOf []", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a list", "[ ]"), - hintfmt("while evaluating the first argument passed to builtins.baseNameOf")); + HintFmt("cannot coerce %s to a string: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the first argument passed to builtins.baseNameOf")); } @@ -377,30 +377,30 @@ namespace nix { TEST_F(ErrorTraceTest, filterSource) { ASSERT_TRACE2("filterSource [] []", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a list", "[ ]"), - hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); + HintFmt("cannot coerce %s to a string: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); ASSERT_TRACE2("filterSource [] \"foo\"", EvalError, - hintfmt("string '%s' doesn't represent an absolute path", "foo"), - hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); + HintFmt("string '%s' doesn't represent an absolute path", "foo"), + HintFmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'")); ASSERT_TRACE2("filterSource [] ./.", TypeError, - hintfmt("expected a function but found %s: %s", "a list", "[ ]"), - hintfmt("while evaluating the first argument passed to builtins.filterSource")); + HintFmt("expected a function but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the first argument passed to builtins.filterSource")); // Usupported by store "dummy" // ASSERT_TRACE2("filterSource (_: 1) ./.", // TypeError, - // hintfmt("attempt to call something which is not a function but %s", "an integer"), - // hintfmt("while adding path '/home/layus/projects/nix'")); + // HintFmt("attempt to call something which is not a function but %s", "an integer"), + // HintFmt("while adding path '/home/layus/projects/nix'")); // ASSERT_TRACE2("filterSource (_: _: 1) ./.", // TypeError, - // hintfmt("expected a Boolean but found %s: %s", "an integer", "1"), - // hintfmt("while evaluating the return value of the path filter function")); + // HintFmt("expected a Boolean but found %s: %s", "an integer", "1"), + // HintFmt("while evaluating the return value of the path filter function")); } @@ -412,8 +412,8 @@ namespace nix { TEST_F(ErrorTraceTest, attrNames) { ASSERT_TRACE2("attrNames []", TypeError, - hintfmt("expected a set but found %s: %s", "a list", "[ ]"), - hintfmt("while evaluating the argument passed to builtins.attrNames")); + HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the argument passed to builtins.attrNames")); } @@ -421,8 +421,8 @@ namespace nix { TEST_F(ErrorTraceTest, attrValues) { ASSERT_TRACE2("attrValues []", TypeError, - hintfmt("expected a set but found %s: %s", "a list", "[ ]"), - hintfmt("while evaluating the argument passed to builtins.attrValues")); + HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the argument passed to builtins.attrValues")); } @@ -430,18 +430,18 @@ namespace nix { TEST_F(ErrorTraceTest, getAttr) { ASSERT_TRACE2("getAttr [] []", TypeError, - hintfmt("expected a string but found %s: %s", "a list", "[ ]"), - hintfmt("while evaluating the first argument passed to builtins.getAttr")); + HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the first argument passed to builtins.getAttr")); ASSERT_TRACE2("getAttr \"foo\" []", TypeError, - hintfmt("expected a set but found %s: %s", "a list", "[ ]"), - hintfmt("while evaluating the second argument passed to builtins.getAttr")); + HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the second argument passed to builtins.getAttr")); ASSERT_TRACE2("getAttr \"foo\" {}", TypeError, - hintfmt("attribute '%s' missing", "foo"), - hintfmt("in the attribute set under consideration")); + HintFmt("attribute '%s' missing", "foo"), + HintFmt("in the attribute set under consideration")); } @@ -453,13 +453,13 @@ namespace nix { TEST_F(ErrorTraceTest, hasAttr) { ASSERT_TRACE2("hasAttr [] []", TypeError, - hintfmt("expected a string but found %s: %s", "a list", "[ ]"), - hintfmt("while evaluating the first argument passed to builtins.hasAttr")); + HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the first argument passed to builtins.hasAttr")); ASSERT_TRACE2("hasAttr \"foo\" []", TypeError, - hintfmt("expected a set but found %s: %s", "a list", "[ ]"), - hintfmt("while evaluating the second argument passed to builtins.hasAttr")); + HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the second argument passed to builtins.hasAttr")); } @@ -471,18 +471,18 @@ namespace nix { TEST_F(ErrorTraceTest, removeAttrs) { ASSERT_TRACE2("removeAttrs \"\" \"\"", TypeError, - hintfmt("expected a set but found %s: %s", "a string", ANSI_MAGENTA "\"\"" ANSI_NORMAL), - hintfmt("while evaluating the first argument passed to builtins.removeAttrs")); + HintFmt("expected a set but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"\"" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.removeAttrs")); ASSERT_TRACE2("removeAttrs \"\" [ 1 ]", TypeError, - hintfmt("expected a set but found %s: %s", "a string", ANSI_MAGENTA "\"\"" ANSI_NORMAL), - hintfmt("while evaluating the first argument passed to builtins.removeAttrs")); + HintFmt("expected a set but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"\"" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.removeAttrs")); ASSERT_TRACE2("removeAttrs \"\" [ \"1\" ]", TypeError, - hintfmt("expected a set but found %s: %s", "a string", ANSI_MAGENTA "\"\"" ANSI_NORMAL), - hintfmt("while evaluating the first argument passed to builtins.removeAttrs")); + HintFmt("expected a set but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"\"" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.removeAttrs")); } @@ -490,28 +490,28 @@ namespace nix { TEST_F(ErrorTraceTest, listToAttrs) { ASSERT_TRACE2("listToAttrs 1", TypeError, - hintfmt("expected a list but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating the argument passed to builtins.listToAttrs")); + HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the argument passed to builtins.listToAttrs")); ASSERT_TRACE2("listToAttrs [ 1 ]", TypeError, - hintfmt("expected a set but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating an element of the list passed to builtins.listToAttrs")); + HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating an element of the list passed to builtins.listToAttrs")); ASSERT_TRACE2("listToAttrs [ {} ]", TypeError, - hintfmt("attribute '%s' missing", "name"), - hintfmt("in a {name=...; value=...;} pair")); + HintFmt("attribute '%s' missing", "name"), + HintFmt("in a {name=...; value=...;} pair")); ASSERT_TRACE2("listToAttrs [ { name = 1; } ]", TypeError, - hintfmt("expected a string but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating the `name` attribute of an element of the list passed to builtins.listToAttrs")); + HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the `name` attribute of an element of the list passed to builtins.listToAttrs")); ASSERT_TRACE2("listToAttrs [ { name = \"foo\"; } ]", TypeError, - hintfmt("attribute '%s' missing", "value"), - hintfmt("in a {name=...; value=...;} pair")); + HintFmt("attribute '%s' missing", "value"), + HintFmt("in a {name=...; value=...;} pair")); } @@ -519,13 +519,13 @@ namespace nix { TEST_F(ErrorTraceTest, intersectAttrs) { ASSERT_TRACE2("intersectAttrs [] []", TypeError, - hintfmt("expected a set but found %s: %s", "a list", "[ ]"), - hintfmt("while evaluating the first argument passed to builtins.intersectAttrs")); + HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the first argument passed to builtins.intersectAttrs")); ASSERT_TRACE2("intersectAttrs {} []", TypeError, - hintfmt("expected a set but found %s: %s", "a list", "[ ]"), - hintfmt("while evaluating the second argument passed to builtins.intersectAttrs")); + HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the second argument passed to builtins.intersectAttrs")); } @@ -533,23 +533,23 @@ namespace nix { TEST_F(ErrorTraceTest, catAttrs) { ASSERT_TRACE2("catAttrs [] {}", TypeError, - hintfmt("expected a string but found %s: %s", "a list", "[ ]"), - hintfmt("while evaluating the first argument passed to builtins.catAttrs")); + HintFmt("expected a string but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the first argument passed to builtins.catAttrs")); ASSERT_TRACE2("catAttrs \"foo\" {}", TypeError, - hintfmt("expected a list but found %s: %s", "a set", "{ }"), - hintfmt("while evaluating the second argument passed to builtins.catAttrs")); + HintFmt("expected a list but found %s: %s", "a set", Uncolored("{ }")), + HintFmt("while evaluating the second argument passed to builtins.catAttrs")); ASSERT_TRACE2("catAttrs \"foo\" [ 1 ]", TypeError, - hintfmt("expected a set but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating an element in the list passed as second argument to builtins.catAttrs")); + HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating an element in the list passed as second argument to builtins.catAttrs")); ASSERT_TRACE2("catAttrs \"foo\" [ { foo = 1; } 1 { bar = 5;} ]", TypeError, - hintfmt("expected a set but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating an element in the list passed as second argument to builtins.catAttrs")); + HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating an element in the list passed as second argument to builtins.catAttrs")); } @@ -557,7 +557,7 @@ namespace nix { TEST_F(ErrorTraceTest, functionArgs) { ASSERT_TRACE1("functionArgs {}", TypeError, - hintfmt("'functionArgs' requires a function")); + HintFmt("'functionArgs' requires a function")); } @@ -565,24 +565,24 @@ namespace nix { TEST_F(ErrorTraceTest, mapAttrs) { ASSERT_TRACE2("mapAttrs [] []", TypeError, - hintfmt("expected a set but found %s: %s", "a list", "[ ]"), - hintfmt("while evaluating the second argument passed to builtins.mapAttrs")); + HintFmt("expected a set but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the second argument passed to builtins.mapAttrs")); // XXX: defered // ASSERT_TRACE2("mapAttrs \"\" { foo.bar = 1; }", // TypeError, - // hintfmt("attempt to call something which is not a function but %s", "a string"), - // hintfmt("while evaluating the attribute 'foo'")); + // HintFmt("attempt to call something which is not a function but %s", "a string"), + // HintFmt("while evaluating the attribute 'foo'")); // ASSERT_TRACE2("mapAttrs (x: x + \"1\") { foo.bar = 1; }", // TypeError, - // hintfmt("attempt to call something which is not a function but %s", "a string"), - // hintfmt("while evaluating the attribute 'foo'")); + // HintFmt("attempt to call something which is not a function but %s", "a string"), + // HintFmt("while evaluating the attribute 'foo'")); // ASSERT_TRACE2("mapAttrs (x: y: x + 1) { foo.bar = 1; }", // TypeError, - // hintfmt("cannot coerce %s to a string", "an integer"), - // hintfmt("while evaluating a path segment")); + // HintFmt("cannot coerce %s to a string", "an integer"), + // HintFmt("while evaluating a path segment")); } @@ -590,27 +590,27 @@ namespace nix { TEST_F(ErrorTraceTest, zipAttrsWith) { ASSERT_TRACE2("zipAttrsWith [] [ 1 ]", TypeError, - hintfmt("expected a function but found %s: %s", "a list", "[ ]"), - hintfmt("while evaluating the first argument passed to builtins.zipAttrsWith")); + HintFmt("expected a function but found %s: %s", "a list", Uncolored("[ ]")), + HintFmt("while evaluating the first argument passed to builtins.zipAttrsWith")); ASSERT_TRACE2("zipAttrsWith (_: 1) [ 1 ]", TypeError, - hintfmt("expected a set but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating a value of the list passed as second argument to builtins.zipAttrsWith")); + HintFmt("expected a set but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating a value of the list passed as second argument to builtins.zipAttrsWith")); // XXX: How to properly tell that the fucntion takes two arguments ? // The same question also applies to sort, and maybe others. // Due to lazyness, we only create a thunk, and it fails later on. // ASSERT_TRACE2("zipAttrsWith (_: 1) [ { foo = 1; } ]", // TypeError, - // hintfmt("attempt to call something which is not a function but %s", "an integer"), - // hintfmt("while evaluating the attribute 'foo'")); + // HintFmt("attempt to call something which is not a function but %s", "an integer"), + // HintFmt("while evaluating the attribute 'foo'")); // XXX: Also deferred deeply // ASSERT_TRACE2("zipAttrsWith (a: b: a + b) [ { foo = 1; } { foo = 2; } ]", // TypeError, - // hintfmt("cannot coerce %s to a string", "a list"), - // hintfmt("while evaluating a path segment")); + // HintFmt("cannot coerce %s to a string", "a list"), + // HintFmt("while evaluating a path segment")); } @@ -622,16 +622,16 @@ namespace nix { TEST_F(ErrorTraceTest, elemAt) { ASSERT_TRACE2("elemAt \"foo\" (-1)", TypeError, - hintfmt("expected a list but found %s: %s", "a string", ANSI_MAGENTA "\"foo\"" ANSI_NORMAL), - hintfmt("while evaluating the first argument passed to builtins.elemAt")); + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.elemAt")); ASSERT_TRACE1("elemAt [] (-1)", Error, - hintfmt("list index %d is out of bounds", -1)); + HintFmt("list index %d is out of bounds", -1)); ASSERT_TRACE1("elemAt [\"foo\"] 3", Error, - hintfmt("list index %d is out of bounds", 3)); + HintFmt("list index %d is out of bounds", 3)); } @@ -639,12 +639,12 @@ namespace nix { TEST_F(ErrorTraceTest, head) { ASSERT_TRACE2("head 1", TypeError, - hintfmt("expected a list but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating the first argument passed to builtins.elemAt")); + HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.elemAt")); ASSERT_TRACE1("head []", Error, - hintfmt("list index %d is out of bounds", 0)); + HintFmt("list index %d is out of bounds", 0)); } @@ -652,12 +652,12 @@ namespace nix { TEST_F(ErrorTraceTest, tail) { ASSERT_TRACE2("tail 1", TypeError, - hintfmt("expected a list but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating the first argument passed to builtins.tail")); + HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.tail")); ASSERT_TRACE1("tail []", Error, - hintfmt("'tail' called on an empty list")); + HintFmt("'tail' called on an empty list")); } @@ -665,13 +665,13 @@ namespace nix { TEST_F(ErrorTraceTest, map) { ASSERT_TRACE2("map 1 \"foo\"", TypeError, - hintfmt("expected a list but found %s: %s", "a string", ANSI_MAGENTA "\"foo\"" ANSI_NORMAL), - hintfmt("while evaluating the second argument passed to builtins.map")); + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.map")); ASSERT_TRACE2("map 1 [ 1 ]", TypeError, - hintfmt("expected a function but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating the first argument passed to builtins.map")); + HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.map")); } @@ -679,18 +679,18 @@ namespace nix { TEST_F(ErrorTraceTest, filter) { ASSERT_TRACE2("filter 1 \"foo\"", TypeError, - hintfmt("expected a list but found %s: %s", "a string", ANSI_MAGENTA "\"foo\"" ANSI_NORMAL), - hintfmt("while evaluating the second argument passed to builtins.filter")); + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.filter")); ASSERT_TRACE2("filter 1 [ \"foo\" ]", TypeError, - hintfmt("expected a function but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating the first argument passed to builtins.filter")); + HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.filter")); ASSERT_TRACE2("filter (_: 5) [ \"foo\" ]", TypeError, - hintfmt("expected a Boolean but found %s: %s", "an integer", ANSI_CYAN "5" ANSI_NORMAL), - hintfmt("while evaluating the return value of the filtering function passed to builtins.filter")); + HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "5" ANSI_NORMAL)), + HintFmt("while evaluating the return value of the filtering function passed to builtins.filter")); } @@ -698,8 +698,8 @@ namespace nix { TEST_F(ErrorTraceTest, elem) { ASSERT_TRACE2("elem 1 \"foo\"", TypeError, - hintfmt("expected a list but found %s: %s", "a string", ANSI_MAGENTA "\"foo\"" ANSI_NORMAL), - hintfmt("while evaluating the second argument passed to builtins.elem")); + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.elem")); } @@ -707,18 +707,18 @@ namespace nix { TEST_F(ErrorTraceTest, concatLists) { ASSERT_TRACE2("concatLists 1", TypeError, - hintfmt("expected a list but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating the first argument passed to builtins.concatLists")); + HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.concatLists")); ASSERT_TRACE2("concatLists [ 1 ]", TypeError, - hintfmt("expected a list but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating a value of the list passed to builtins.concatLists")); + HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating a value of the list passed to builtins.concatLists")); ASSERT_TRACE2("concatLists [ [1] \"foo\" ]", TypeError, - hintfmt("expected a list but found %s: %s", "a string", ANSI_MAGENTA "\"foo\"" ANSI_NORMAL), - hintfmt("while evaluating a value of the list passed to builtins.concatLists")); + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating a value of the list passed to builtins.concatLists")); } @@ -726,13 +726,13 @@ namespace nix { TEST_F(ErrorTraceTest, length) { ASSERT_TRACE2("length 1", TypeError, - hintfmt("expected a list but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating the first argument passed to builtins.length")); + HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.length")); ASSERT_TRACE2("length \"foo\"", TypeError, - hintfmt("expected a list but found %s: %s", "a string", ANSI_MAGENTA "\"foo\"" ANSI_NORMAL), - hintfmt("while evaluating the first argument passed to builtins.length")); + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.length")); } @@ -740,22 +740,22 @@ namespace nix { TEST_F(ErrorTraceTest, foldlPrime) { ASSERT_TRACE2("foldl' 1 \"foo\" true", TypeError, - hintfmt("expected a function but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating the first argument passed to builtins.foldlStrict")); + HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.foldlStrict")); ASSERT_TRACE2("foldl' (_: 1) \"foo\" true", TypeError, - hintfmt("expected a list but found %s: %s", "a Boolean", ANSI_CYAN "true" ANSI_NORMAL), - hintfmt("while evaluating the third argument passed to builtins.foldlStrict")); + HintFmt("expected a list but found %s: %s", "a Boolean", Uncolored(ANSI_CYAN "true" ANSI_NORMAL)), + HintFmt("while evaluating the third argument passed to builtins.foldlStrict")); ASSERT_TRACE1("foldl' (_: 1) \"foo\" [ true ]", TypeError, - hintfmt("attempt to call something which is not a function but %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL)); + HintFmt("attempt to call something which is not a function but %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL))); ASSERT_TRACE2("foldl' (a: b: a && b) \"foo\" [ true ]", TypeError, - hintfmt("expected a Boolean but found %s: %s", "a string", ANSI_MAGENTA "\"foo\"" ANSI_NORMAL), - hintfmt("in the left operand of the AND (&&) operator")); + HintFmt("expected a Boolean but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("in the left operand of the AND (&&) operator")); } @@ -763,18 +763,18 @@ namespace nix { TEST_F(ErrorTraceTest, any) { ASSERT_TRACE2("any 1 \"foo\"", TypeError, - hintfmt("expected a function but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating the first argument passed to builtins.any")); + HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.any")); ASSERT_TRACE2("any (_: 1) \"foo\"", TypeError, - hintfmt("expected a list but found %s: %s", "a string", ANSI_MAGENTA "\"foo\"" ANSI_NORMAL), - hintfmt("while evaluating the second argument passed to builtins.any")); + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.any")); ASSERT_TRACE2("any (_: 1) [ \"foo\" ]", TypeError, - hintfmt("expected a Boolean but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating the return value of the function passed to builtins.any")); + HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the return value of the function passed to builtins.any")); } @@ -782,18 +782,18 @@ namespace nix { TEST_F(ErrorTraceTest, all) { ASSERT_TRACE2("all 1 \"foo\"", TypeError, - hintfmt("expected a function but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating the first argument passed to builtins.all")); + HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.all")); ASSERT_TRACE2("all (_: 1) \"foo\"", TypeError, - hintfmt("expected a list but found %s: %s", "a string", ANSI_MAGENTA "\"foo\"" ANSI_NORMAL), - hintfmt("while evaluating the second argument passed to builtins.all")); + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.all")); ASSERT_TRACE2("all (_: 1) [ \"foo\" ]", TypeError, - hintfmt("expected a Boolean but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating the return value of the function passed to builtins.all")); + HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the return value of the function passed to builtins.all")); } @@ -801,23 +801,23 @@ namespace nix { TEST_F(ErrorTraceTest, genList) { ASSERT_TRACE2("genList 1 \"foo\"", TypeError, - hintfmt("expected an integer but found %s: %s", "a string", ANSI_MAGENTA "\"foo\"" ANSI_NORMAL), - hintfmt("while evaluating the second argument passed to builtins.genList")); + HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.genList")); ASSERT_TRACE2("genList 1 2", TypeError, - hintfmt("expected a function but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating the first argument passed to builtins.genList", "an integer")); + HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.genList")); // XXX: defered // ASSERT_TRACE2("genList (x: x + \"foo\") 2 #TODO", // TypeError, - // hintfmt("cannot add %s to an integer", "a string"), - // hintfmt("while evaluating anonymous lambda")); + // HintFmt("cannot add %s to an integer", "a string"), + // HintFmt("while evaluating anonymous lambda")); ASSERT_TRACE1("genList false (-3)", EvalError, - hintfmt("cannot create list of size %d", -3)); + HintFmt("cannot create list of size %d", -3)); } @@ -825,31 +825,31 @@ namespace nix { TEST_F(ErrorTraceTest, sort) { ASSERT_TRACE2("sort 1 \"foo\"", TypeError, - hintfmt("expected a list but found %s: %s", "a string", ANSI_MAGENTA "\"foo\"" ANSI_NORMAL), - hintfmt("while evaluating the second argument passed to builtins.sort")); + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.sort")); ASSERT_TRACE2("sort 1 [ \"foo\" ]", TypeError, - hintfmt("expected a function but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating the first argument passed to builtins.sort")); + HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.sort")); ASSERT_TRACE1("sort (_: 1) [ \"foo\" \"bar\" ]", TypeError, - hintfmt("attempt to call something which is not a function but %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL)); + HintFmt("attempt to call something which is not a function but %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL))); ASSERT_TRACE2("sort (_: _: 1) [ \"foo\" \"bar\" ]", TypeError, - hintfmt("expected a Boolean but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating the return value of the sorting function passed to builtins.sort")); + HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the return value of the sorting function passed to builtins.sort")); // XXX: Trace too deep, need better asserts // ASSERT_TRACE1("sort (a: b: a <= b) [ \"foo\" {} ] # TODO", // TypeError, - // hintfmt("cannot compare %s with %s", "a string", "a set")); + // HintFmt("cannot compare %s with %s", "a string", "a set")); // ASSERT_TRACE1("sort (a: b: a <= b) [ {} {} ] # TODO", // TypeError, - // hintfmt("cannot compare %s with %s; values of that type are incomparable", "a set", "a set")); + // HintFmt("cannot compare %s with %s; values of that type are incomparable", "a set", "a set")); } @@ -857,18 +857,18 @@ namespace nix { TEST_F(ErrorTraceTest, partition) { ASSERT_TRACE2("partition 1 \"foo\"", TypeError, - hintfmt("expected a function but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating the first argument passed to builtins.partition")); + HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.partition")); ASSERT_TRACE2("partition (_: 1) \"foo\"", TypeError, - hintfmt("expected a list but found %s: %s", "a string", ANSI_MAGENTA "\"foo\"" ANSI_NORMAL), - hintfmt("while evaluating the second argument passed to builtins.partition")); + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.partition")); ASSERT_TRACE2("partition (_: 1) [ \"foo\" ]", TypeError, - hintfmt("expected a Boolean but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating the return value of the partition function passed to builtins.partition")); + HintFmt("expected a Boolean but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the return value of the partition function passed to builtins.partition")); } @@ -876,18 +876,18 @@ namespace nix { TEST_F(ErrorTraceTest, groupBy) { ASSERT_TRACE2("groupBy 1 \"foo\"", TypeError, - hintfmt("expected a function but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating the first argument passed to builtins.groupBy")); + HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.groupBy")); ASSERT_TRACE2("groupBy (_: 1) \"foo\"", TypeError, - hintfmt("expected a list but found %s: %s", "a string", ANSI_MAGENTA "\"foo\"" ANSI_NORMAL), - hintfmt("while evaluating the second argument passed to builtins.groupBy")); + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.groupBy")); ASSERT_TRACE2("groupBy (x: x) [ \"foo\" \"bar\" 1 ]", TypeError, - hintfmt("expected a string but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating the return value of the grouping function passed to builtins.groupBy")); + HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the return value of the grouping function passed to builtins.groupBy")); } @@ -895,23 +895,23 @@ namespace nix { TEST_F(ErrorTraceTest, concatMap) { ASSERT_TRACE2("concatMap 1 \"foo\"", TypeError, - hintfmt("expected a function but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating the first argument passed to builtins.concatMap")); + HintFmt("expected a function but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.concatMap")); ASSERT_TRACE2("concatMap (x: 1) \"foo\"", TypeError, - hintfmt("expected a list but found %s: %s", "a string", ANSI_MAGENTA "\"foo\"" ANSI_NORMAL), - hintfmt("while evaluating the second argument passed to builtins.concatMap")); + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.concatMap")); ASSERT_TRACE2("concatMap (x: 1) [ \"foo\" ] # TODO", TypeError, - hintfmt("expected a list but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating the return value of the function passed to builtins.concatMap")); + HintFmt("expected a list but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the return value of the function passed to builtins.concatMap")); ASSERT_TRACE2("concatMap (x: \"foo\") [ 1 2 ] # TODO", TypeError, - hintfmt("expected a list but found %s: %s", "a string", ANSI_MAGENTA "\"foo\"" ANSI_NORMAL), - hintfmt("while evaluating the return value of the function passed to builtins.concatMap")); + HintFmt("expected a list but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the return value of the function passed to builtins.concatMap")); } @@ -919,13 +919,13 @@ namespace nix { TEST_F(ErrorTraceTest, add) { ASSERT_TRACE2("add \"foo\" 1", TypeError, - hintfmt("expected an integer but found %s: %s", "a string", ANSI_MAGENTA "\"foo\"" ANSI_NORMAL), - hintfmt("while evaluating the first argument of the addition")); + HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the first argument of the addition")); ASSERT_TRACE2("add 1 \"foo\"", TypeError, - hintfmt("expected an integer but found %s: %s", "a string", ANSI_MAGENTA "\"foo\"" ANSI_NORMAL), - hintfmt("while evaluating the second argument of the addition")); + HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument of the addition")); } @@ -933,13 +933,13 @@ namespace nix { TEST_F(ErrorTraceTest, sub) { ASSERT_TRACE2("sub \"foo\" 1", TypeError, - hintfmt("expected an integer but found %s: %s", "a string", ANSI_MAGENTA "\"foo\"" ANSI_NORMAL), - hintfmt("while evaluating the first argument of the subtraction")); + HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the first argument of the subtraction")); ASSERT_TRACE2("sub 1 \"foo\"", TypeError, - hintfmt("expected an integer but found %s: %s", "a string", ANSI_MAGENTA "\"foo\"" ANSI_NORMAL), - hintfmt("while evaluating the second argument of the subtraction")); + HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument of the subtraction")); } @@ -947,13 +947,13 @@ namespace nix { TEST_F(ErrorTraceTest, mul) { ASSERT_TRACE2("mul \"foo\" 1", TypeError, - hintfmt("expected an integer but found %s: %s", "a string", ANSI_MAGENTA "\"foo\"" ANSI_NORMAL), - hintfmt("while evaluating the first argument of the multiplication")); + HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the first argument of the multiplication")); ASSERT_TRACE2("mul 1 \"foo\"", TypeError, - hintfmt("expected an integer but found %s: %s", "a string", ANSI_MAGENTA "\"foo\"" ANSI_NORMAL), - hintfmt("while evaluating the second argument of the multiplication")); + HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument of the multiplication")); } @@ -961,17 +961,17 @@ namespace nix { TEST_F(ErrorTraceTest, div) { ASSERT_TRACE2("div \"foo\" 1 # TODO: an integer was expected -> a number", TypeError, - hintfmt("expected an integer but found %s: %s", "a string", ANSI_MAGENTA "\"foo\"" ANSI_NORMAL), - hintfmt("while evaluating the first operand of the division")); + HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the first operand of the division")); ASSERT_TRACE2("div 1 \"foo\"", TypeError, - hintfmt("expected a float but found %s: %s", "a string", ANSI_MAGENTA "\"foo\"" ANSI_NORMAL), - hintfmt("while evaluating the second operand of the division")); + HintFmt("expected a float but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second operand of the division")); ASSERT_TRACE1("div \"foo\" 0", EvalError, - hintfmt("division by zero")); + HintFmt("division by zero")); } @@ -979,13 +979,13 @@ namespace nix { TEST_F(ErrorTraceTest, bitAnd) { ASSERT_TRACE2("bitAnd 1.1 2", TypeError, - hintfmt("expected an integer but found %s: %s", "a float", ANSI_CYAN "1.1" ANSI_NORMAL), - hintfmt("while evaluating the first argument passed to builtins.bitAnd")); + HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "1.1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.bitAnd")); ASSERT_TRACE2("bitAnd 1 2.2", TypeError, - hintfmt("expected an integer but found %s: %s", "a float", ANSI_CYAN "2.2" ANSI_NORMAL), - hintfmt("while evaluating the second argument passed to builtins.bitAnd")); + HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "2.2" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.bitAnd")); } @@ -993,13 +993,13 @@ namespace nix { TEST_F(ErrorTraceTest, bitOr) { ASSERT_TRACE2("bitOr 1.1 2", TypeError, - hintfmt("expected an integer but found %s: %s", "a float", ANSI_CYAN "1.1" ANSI_NORMAL), - hintfmt("while evaluating the first argument passed to builtins.bitOr")); + HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "1.1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.bitOr")); ASSERT_TRACE2("bitOr 1 2.2", TypeError, - hintfmt("expected an integer but found %s: %s", "a float", ANSI_CYAN "2.2" ANSI_NORMAL), - hintfmt("while evaluating the second argument passed to builtins.bitOr")); + HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "2.2" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.bitOr")); } @@ -1007,13 +1007,13 @@ namespace nix { TEST_F(ErrorTraceTest, bitXor) { ASSERT_TRACE2("bitXor 1.1 2", TypeError, - hintfmt("expected an integer but found %s: %s", "a float", ANSI_CYAN "1.1" ANSI_NORMAL), - hintfmt("while evaluating the first argument passed to builtins.bitXor")); + HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "1.1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.bitXor")); ASSERT_TRACE2("bitXor 1 2.2", TypeError, - hintfmt("expected an integer but found %s: %s", "a float", ANSI_CYAN "2.2" ANSI_NORMAL), - hintfmt("while evaluating the second argument passed to builtins.bitXor")); + HintFmt("expected an integer but found %s: %s", "a float", Uncolored(ANSI_CYAN "2.2" ANSI_NORMAL)), + HintFmt("while evaluating the second argument passed to builtins.bitXor")); } @@ -1021,16 +1021,16 @@ namespace nix { TEST_F(ErrorTraceTest, lessThan) { ASSERT_TRACE1("lessThan 1 \"foo\"", EvalError, - hintfmt("cannot compare %s with %s", "an integer", "a string")); + HintFmt("cannot compare %s with %s", "an integer", "a string")); ASSERT_TRACE1("lessThan {} {}", EvalError, - hintfmt("cannot compare %s with %s; values of that type are incomparable", "a set", "a set")); + HintFmt("cannot compare %s with %s; values of that type are incomparable", "a set", "a set")); ASSERT_TRACE2("lessThan [ 1 2 ] [ \"foo\" ]", EvalError, - hintfmt("cannot compare %s with %s", "an integer", "a string"), - hintfmt("while comparing two list elements")); + HintFmt("cannot compare %s with %s", "an integer", "a string"), + HintFmt("while comparing two list elements")); } @@ -1038,8 +1038,8 @@ namespace nix { TEST_F(ErrorTraceTest, toString) { ASSERT_TRACE2("toString { a = 1; }", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a set", "{ a = " ANSI_CYAN "1" ANSI_NORMAL "; }"), - hintfmt("while evaluating the first argument passed to builtins.toString")); + HintFmt("cannot coerce %s to a string: %s", "a set", Uncolored("{ a = " ANSI_CYAN "1" ANSI_NORMAL "; }")), + HintFmt("while evaluating the first argument passed to builtins.toString")); } @@ -1047,22 +1047,22 @@ namespace nix { TEST_F(ErrorTraceTest, substring) { ASSERT_TRACE2("substring {} \"foo\" true", TypeError, - hintfmt("expected an integer but found %s: %s", "a set", "{ }"), - hintfmt("while evaluating the first argument (the start offset) passed to builtins.substring")); + HintFmt("expected an integer but found %s: %s", "a set", Uncolored("{ }")), + HintFmt("while evaluating the first argument (the start offset) passed to builtins.substring")); ASSERT_TRACE2("substring 3 \"foo\" true", TypeError, - hintfmt("expected an integer but found %s: %s", "a string", ANSI_MAGENTA "\"foo\"" ANSI_NORMAL), - hintfmt("while evaluating the second argument (the substring length) passed to builtins.substring")); + HintFmt("expected an integer but found %s: %s", "a string", Uncolored(ANSI_MAGENTA "\"foo\"" ANSI_NORMAL)), + HintFmt("while evaluating the second argument (the substring length) passed to builtins.substring")); ASSERT_TRACE2("substring 0 3 {}", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"), - hintfmt("while evaluating the third argument (the string) passed to builtins.substring")); + HintFmt("cannot coerce %s to a string: %s", "a set", Uncolored("{ }")), + HintFmt("while evaluating the third argument (the string) passed to builtins.substring")); ASSERT_TRACE1("substring (-3) 3 \"sometext\"", EvalError, - hintfmt("negative start position in 'substring'")); + HintFmt("negative start position in 'substring'")); } @@ -1070,8 +1070,8 @@ namespace nix { TEST_F(ErrorTraceTest, stringLength) { ASSERT_TRACE2("stringLength {} # TODO: context is missing ???", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"), - hintfmt("while evaluating the argument passed to builtins.stringLength")); + HintFmt("cannot coerce %s to a string: %s", "a set", Uncolored("{ }")), + HintFmt("while evaluating the argument passed to builtins.stringLength")); } @@ -1079,17 +1079,17 @@ namespace nix { TEST_F(ErrorTraceTest, hashString) { ASSERT_TRACE2("hashString 1 {}", TypeError, - hintfmt("expected a string but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating the first argument passed to builtins.hashString")); + HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.hashString")); ASSERT_TRACE1("hashString \"foo\" \"content\"", UsageError, - hintfmt("unknown hash algorithm '%s', expect 'md5', 'sha1', 'sha256', or 'sha512'", "foo")); + HintFmt("unknown hash algorithm '%s', expect 'md5', 'sha1', 'sha256', or 'sha512'", "foo")); ASSERT_TRACE2("hashString \"sha256\" {}", TypeError, - hintfmt("expected a string but found %s: %s", "a set", "{ }"), - hintfmt("while evaluating the second argument passed to builtins.hashString")); + HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), + HintFmt("while evaluating the second argument passed to builtins.hashString")); } @@ -1097,17 +1097,17 @@ namespace nix { TEST_F(ErrorTraceTest, match) { ASSERT_TRACE2("match 1 {}", TypeError, - hintfmt("expected a string but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating the first argument passed to builtins.match")); + HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.match")); ASSERT_TRACE2("match \"foo\" {}", TypeError, - hintfmt("expected a string but found %s: %s", "a set", "{ }"), - hintfmt("while evaluating the second argument passed to builtins.match")); + HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), + HintFmt("while evaluating the second argument passed to builtins.match")); ASSERT_TRACE1("match \"(.*\" \"\"", EvalError, - hintfmt("invalid regular expression '%s'", "(.*")); + HintFmt("invalid regular expression '%s'", "(.*")); } @@ -1115,17 +1115,17 @@ namespace nix { TEST_F(ErrorTraceTest, split) { ASSERT_TRACE2("split 1 {}", TypeError, - hintfmt("expected a string but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating the first argument passed to builtins.split")); + HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.split")); ASSERT_TRACE2("split \"foo\" {}", TypeError, - hintfmt("expected a string but found %s: %s", "a set", "{ }"), - hintfmt("while evaluating the second argument passed to builtins.split")); + HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), + HintFmt("while evaluating the second argument passed to builtins.split")); ASSERT_TRACE1("split \"f(o*o\" \"1foo2\"", EvalError, - hintfmt("invalid regular expression '%s'", "f(o*o")); + HintFmt("invalid regular expression '%s'", "f(o*o")); } @@ -1133,18 +1133,18 @@ namespace nix { TEST_F(ErrorTraceTest, concatStringsSep) { ASSERT_TRACE2("concatStringsSep 1 {}", TypeError, - hintfmt("expected a string but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating the first argument (the separator string) passed to builtins.concatStringsSep")); + HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument (the separator string) passed to builtins.concatStringsSep")); ASSERT_TRACE2("concatStringsSep \"foo\" {}", TypeError, - hintfmt("expected a list but found %s: %s", "a set", "{ }"), - hintfmt("while evaluating the second argument (the list of strings to concat) passed to builtins.concatStringsSep")); + HintFmt("expected a list but found %s: %s", "a set", Uncolored("{ }")), + HintFmt("while evaluating the second argument (the list of strings to concat) passed to builtins.concatStringsSep")); ASSERT_TRACE2("concatStringsSep \"foo\" [ 1 2 {} ] # TODO: coerce to string is buggy", TypeError, - hintfmt("cannot coerce %s to a string: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating one element of the list of strings to concat passed to builtins.concatStringsSep")); + HintFmt("cannot coerce %s to a string: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating one element of the list of strings to concat passed to builtins.concatStringsSep")); } @@ -1152,8 +1152,8 @@ namespace nix { TEST_F(ErrorTraceTest, parseDrvName) { ASSERT_TRACE2("parseDrvName 1", TypeError, - hintfmt("expected a string but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating the first argument passed to builtins.parseDrvName")); + HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.parseDrvName")); } @@ -1161,13 +1161,13 @@ namespace nix { TEST_F(ErrorTraceTest, compareVersions) { ASSERT_TRACE2("compareVersions 1 {}", TypeError, - hintfmt("expected a string but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating the first argument passed to builtins.compareVersions")); + HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.compareVersions")); ASSERT_TRACE2("compareVersions \"abd\" {}", TypeError, - hintfmt("expected a string but found %s: %s", "a set", "{ }"), - hintfmt("while evaluating the second argument passed to builtins.compareVersions")); + HintFmt("expected a string but found %s: %s", "a set", Uncolored("{ }")), + HintFmt("while evaluating the second argument passed to builtins.compareVersions")); } @@ -1175,8 +1175,8 @@ namespace nix { TEST_F(ErrorTraceTest, splitVersion) { ASSERT_TRACE2("splitVersion 1", TypeError, - hintfmt("expected a string but found %s: %s", "an integer", ANSI_CYAN "1" ANSI_NORMAL), - hintfmt("while evaluating the first argument passed to builtins.splitVersion")); + HintFmt("expected a string but found %s: %s", "an integer", Uncolored(ANSI_CYAN "1" ANSI_NORMAL)), + HintFmt("while evaluating the first argument passed to builtins.splitVersion")); } @@ -1189,108 +1189,108 @@ namespace nix { TEST_F(ErrorTraceTest, derivationStrict) { ASSERT_TRACE2("derivationStrict \"\"", TypeError, - hintfmt("expected a set but found %s: %s", "a string", "\"\""), - hintfmt("while evaluating the argument passed to builtins.derivationStrict")); + HintFmt("expected a set but found %s: %s", "a string", "\"\""), + HintFmt("while evaluating the argument passed to builtins.derivationStrict")); ASSERT_TRACE2("derivationStrict {}", TypeError, - hintfmt("attribute '%s' missing", "name"), - hintfmt("in the attrset passed as argument to builtins.derivationStrict")); + HintFmt("attribute '%s' missing", "name"), + HintFmt("in the attrset passed as argument to builtins.derivationStrict")); ASSERT_TRACE2("derivationStrict { name = 1; }", TypeError, - hintfmt("expected a string but found %s: %s", "an integer", "1"), - hintfmt("while evaluating the `name` attribute passed to builtins.derivationStrict")); + HintFmt("expected a string but found %s: %s", "an integer", "1"), + HintFmt("while evaluating the `name` attribute passed to builtins.derivationStrict")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; }", TypeError, - hintfmt("required attribute 'builder' missing"), - hintfmt("while evaluating derivation 'foo'")); + HintFmt("required attribute 'builder' missing"), + HintFmt("while evaluating derivation 'foo'")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; __structuredAttrs = 15; }", TypeError, - hintfmt("expected a Boolean but found %s: %s", "an integer", "15"), - hintfmt("while evaluating the `__structuredAttrs` attribute passed to builtins.derivationStrict")); + HintFmt("expected a Boolean but found %s: %s", "an integer", "15"), + HintFmt("while evaluating the `__structuredAttrs` attribute passed to builtins.derivationStrict")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; __ignoreNulls = 15; }", TypeError, - hintfmt("expected a Boolean but found %s: %s", "an integer", "15"), - hintfmt("while evaluating the `__ignoreNulls` attribute passed to builtins.derivationStrict")); + HintFmt("expected a Boolean but found %s: %s", "an integer", "15"), + HintFmt("while evaluating the `__ignoreNulls` attribute passed to builtins.derivationStrict")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; outputHashMode = 15; }", TypeError, - hintfmt("invalid value '15' for 'outputHashMode' attribute"), - hintfmt("while evaluating the attribute 'outputHashMode' of derivation 'foo'")); + HintFmt("invalid value '15' for 'outputHashMode' attribute"), + HintFmt("while evaluating the attribute 'outputHashMode' of derivation 'foo'")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; outputHashMode = \"custom\"; }", TypeError, - hintfmt("invalid value 'custom' for 'outputHashMode' attribute"), - hintfmt("while evaluating the attribute 'outputHashMode' of derivation 'foo'")); + HintFmt("invalid value 'custom' for 'outputHashMode' attribute"), + HintFmt("while evaluating the attribute 'outputHashMode' of derivation 'foo'")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = {}; }", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"), - hintfmt("while evaluating the attribute 'system' of derivation 'foo'")); + HintFmt("cannot coerce %s to a string: %s", "a set", "{ }"), + HintFmt("while evaluating the attribute 'system' of derivation 'foo'")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = {}; }", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"), - hintfmt("while evaluating the attribute 'outputs' of derivation 'foo'")); + HintFmt("cannot coerce %s to a string: %s", "a set", "{ }"), + HintFmt("while evaluating the attribute 'outputs' of derivation 'foo'")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"drv\"; }", TypeError, - hintfmt("invalid derivation output name 'drv'"), - hintfmt("while evaluating the attribute 'outputs' of derivation 'foo'")); + HintFmt("invalid derivation output name 'drv'"), + HintFmt("while evaluating the attribute 'outputs' of derivation 'foo'")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = []; }", TypeError, - hintfmt("derivation cannot have an empty set of outputs"), - hintfmt("while evaluating the attribute 'outputs' of derivation 'foo'")); + HintFmt("derivation cannot have an empty set of outputs"), + HintFmt("while evaluating the attribute 'outputs' of derivation 'foo'")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = [ \"drv\" ]; }", TypeError, - hintfmt("invalid derivation output name 'drv'"), - hintfmt("while evaluating the attribute 'outputs' of derivation 'foo'")); + HintFmt("invalid derivation output name 'drv'"), + HintFmt("while evaluating the attribute 'outputs' of derivation 'foo'")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = [ \"out\" \"out\" ]; }", TypeError, - hintfmt("duplicate derivation output 'out'"), - hintfmt("while evaluating the attribute 'outputs' of derivation 'foo'")); + HintFmt("duplicate derivation output 'out'"), + HintFmt("while evaluating the attribute 'outputs' of derivation 'foo'")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; __contentAddressed = \"true\"; }", TypeError, - hintfmt("expected a Boolean but found %s: %s", "a string", "\"true\""), - hintfmt("while evaluating the attribute '__contentAddressed' of derivation 'foo'")); + HintFmt("expected a Boolean but found %s: %s", "a string", "\"true\""), + HintFmt("while evaluating the attribute '__contentAddressed' of derivation 'foo'")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; __impure = \"true\"; }", TypeError, - hintfmt("expected a Boolean but found %s: %s", "a string", "\"true\""), - hintfmt("while evaluating the attribute '__impure' of derivation 'foo'")); + HintFmt("expected a Boolean but found %s: %s", "a string", "\"true\""), + HintFmt("while evaluating the attribute '__impure' of derivation 'foo'")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; __impure = \"true\"; }", TypeError, - hintfmt("expected a Boolean but found %s: %s", "a string", "\"true\""), - hintfmt("while evaluating the attribute '__impure' of derivation 'foo'")); + HintFmt("expected a Boolean but found %s: %s", "a string", "\"true\""), + HintFmt("while evaluating the attribute '__impure' of derivation 'foo'")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = \"foo\"; }", TypeError, - hintfmt("expected a list but found %s: %s", "a string", "\"foo\""), - hintfmt("while evaluating the attribute 'args' of derivation 'foo'")); + HintFmt("expected a list but found %s: %s", "a string", "\"foo\""), + HintFmt("while evaluating the attribute 'args' of derivation 'foo'")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ {} ]; }", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"), - hintfmt("while evaluating an element of the argument list")); + HintFmt("cannot coerce %s to a string: %s", "a set", "{ }"), + HintFmt("while evaluating an element of the argument list")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; args = [ \"a\" {} ]; }", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"), - hintfmt("while evaluating an element of the argument list")); + HintFmt("cannot coerce %s to a string: %s", "a set", "{ }"), + HintFmt("while evaluating an element of the argument list")); ASSERT_TRACE2("derivationStrict { name = \"foo\"; builder = 1; system = 1; outputs = \"out\"; FOO = {}; }", TypeError, - hintfmt("cannot coerce %s to a string: %s", "a set", "{ }"), - hintfmt("while evaluating the attribute 'FOO' of derivation 'foo'")); + HintFmt("cannot coerce %s to a string: %s", "a set", "{ }"), + HintFmt("while evaluating the attribute 'FOO' of derivation 'foo'")); } */ diff --git a/tests/unit/libexpr/flake/url-name.cc b/tests/unit/libexpr/flake/url-name.cc index 85387b323..15bc6b111 100644 --- a/tests/unit/libexpr/flake/url-name.cc +++ b/tests/unit/libexpr/flake/url-name.cc @@ -14,6 +14,7 @@ namespace nix { ASSERT_EQ(getNameFromURL(parseURL("path:./repos/myflake#nonStandardAttr.mylaptop")), "mylaptop"); ASSERT_EQ(getNameFromURL(parseURL("path:./nixpkgs#packages.x86_64-linux.complex^bin,man")), "complex"); ASSERT_EQ(getNameFromURL(parseURL("path:./myproj#packages.x86_64-linux.default^*")), "myproj"); + ASSERT_EQ(getNameFromURL(parseURL("path:./myproj#defaultPackage.x86_64-linux")), "myproj"); ASSERT_EQ(getNameFromURL(parseURL("github:NixOS/nixpkgs#packages.x86_64-linux.hello")), "hello"); ASSERT_EQ(getNameFromURL(parseURL("github:NixOS/nixpkgs#hello")), "hello"); diff --git a/tests/unit/libexpr/value/print.cc b/tests/unit/libexpr/value/print.cc index c4264a38d..aabf156c2 100644 --- a/tests/unit/libexpr/value/print.cc +++ b/tests/unit/libexpr/value/print.cc @@ -460,19 +460,7 @@ TEST_F(ValuePrintingTests, ansiColorsError) test(vError, ANSI_RED - "«" - ANSI_RED - "error:" - ANSI_NORMAL - "\n … while calling the '" - ANSI_MAGENTA - "throw" - ANSI_NORMAL - "' builtin\n\n " - ANSI_RED - "error:" - ANSI_NORMAL - " uh oh!»" + "«error: uh oh!»" ANSI_NORMAL, PrintOptions { .ansiColors = true, @@ -501,19 +489,7 @@ TEST_F(ValuePrintingTests, ansiColorsDerivationError) test(vAttrs, "{ drvPath = " ANSI_RED - "«" - ANSI_RED - "error:" - ANSI_NORMAL - "\n … while calling the '" - ANSI_MAGENTA - "throw" - ANSI_NORMAL - "' builtin\n\n " - ANSI_RED - "error:" - ANSI_NORMAL - " uh oh!»" + "«error: uh oh!»" ANSI_NORMAL "; type = " ANSI_MAGENTA @@ -527,19 +503,7 @@ TEST_F(ValuePrintingTests, ansiColorsDerivationError) test(vAttrs, ANSI_RED - "«" - ANSI_RED - "error:" - ANSI_NORMAL - "\n … while calling the '" - ANSI_MAGENTA - "throw" - ANSI_NORMAL - "' builtin\n\n " - ANSI_RED - "error:" - ANSI_NORMAL - " uh oh!»" + "«error: uh oh!»" ANSI_NORMAL, PrintOptions { .ansiColors = true, @@ -560,7 +524,7 @@ TEST_F(ValuePrintingTests, ansiColorsAssert) state.mkThunk_(v, &expr); test(v, - ANSI_RED "«" ANSI_RED "error:" ANSI_NORMAL " assertion '" ANSI_MAGENTA "false" ANSI_NORMAL "' failed»" ANSI_NORMAL, + ANSI_RED "«error: assertion 'false' failed»" ANSI_NORMAL, PrintOptions { .ansiColors = true, .force = true @@ -756,7 +720,7 @@ TEST_F(ValuePrintingTests, ansiColorsAttrsElided) vAttrs.mkAttrs(builder.finish()); test(vAttrs, - "{ one = " ANSI_CYAN "1" ANSI_NORMAL "; " ANSI_FAINT "«1 attribute elided»" ANSI_NORMAL "}", + "{ one = " ANSI_CYAN "1" ANSI_NORMAL "; " ANSI_FAINT "«1 attribute elided»" ANSI_NORMAL " }", PrintOptions { .ansiColors = true, .maxAttrs = 1 @@ -769,7 +733,7 @@ TEST_F(ValuePrintingTests, ansiColorsAttrsElided) vAttrs.mkAttrs(builder.finish()); test(vAttrs, - "{ one = " ANSI_CYAN "1" ANSI_NORMAL "; " ANSI_FAINT "«2 attributes elided»" ANSI_NORMAL "}", + "{ one = " ANSI_CYAN "1" ANSI_NORMAL "; " ANSI_FAINT "«2 attributes elided»" ANSI_NORMAL " }", PrintOptions { .ansiColors = true, .maxAttrs = 1 @@ -793,7 +757,7 @@ TEST_F(ValuePrintingTests, ansiColorsListElided) vList.bigList.size = 2; test(vList, - "[ " ANSI_CYAN "1" ANSI_NORMAL " " ANSI_FAINT "«1 item elided»" ANSI_NORMAL "]", + "[ " ANSI_CYAN "1" ANSI_NORMAL " " ANSI_FAINT "«1 item elided»" ANSI_NORMAL " ]", PrintOptions { .ansiColors = true, .maxListItems = 1 @@ -806,7 +770,7 @@ TEST_F(ValuePrintingTests, ansiColorsListElided) vList.bigList.size = 3; test(vList, - "[ " ANSI_CYAN "1" ANSI_NORMAL " " ANSI_FAINT "«2 items elided»" ANSI_NORMAL "]", + "[ " ANSI_CYAN "1" ANSI_NORMAL " " ANSI_FAINT "«2 items elided»" ANSI_NORMAL " ]", PrintOptions { .ansiColors = true, .maxListItems = 1 diff --git a/tests/unit/libstore/content-address.cc b/tests/unit/libstore/content-address.cc new file mode 100644 index 000000000..cc1c7fcc6 --- /dev/null +++ b/tests/unit/libstore/content-address.cc @@ -0,0 +1,37 @@ +#include + +#include "content-address.hh" + +namespace nix { + +/* ---------------------------------------------------------------------------- + * ContentAddressMethod::parse, ContentAddressMethod::render + * --------------------------------------------------------------------------*/ + +TEST(ContentAddressMethod, testRoundTripPrintParse_1) { + for (const ContentAddressMethod & cam : { + ContentAddressMethod { TextIngestionMethod {} }, + ContentAddressMethod { FileIngestionMethod::Flat }, + ContentAddressMethod { FileIngestionMethod::Recursive }, + ContentAddressMethod { FileIngestionMethod::Git }, + }) { + EXPECT_EQ(ContentAddressMethod::parse(cam.render()), cam); + } +} + +TEST(ContentAddressMethod, testRoundTripPrintParse_2) { + for (const std::string_view camS : { + "text", + "flat", + "nar", + "git", + }) { + EXPECT_EQ(ContentAddressMethod::parse(camS).render(), camS); + } +} + +TEST(ContentAddressMethod, testParseContentAddressMethodOptException) { + EXPECT_THROW(ContentAddressMethod::parse("narwhal"), UsageError); +} + +} diff --git a/tests/unit/libutil/canon-path.cc b/tests/unit/libutil/canon-path.cc index fc94ccc3d..7f91308af 100644 --- a/tests/unit/libutil/canon-path.cc +++ b/tests/unit/libutil/canon-path.cc @@ -41,6 +41,24 @@ namespace nix { } } + TEST(CanonPath, from_existing) { + CanonPath p0("foo//bar/"); + { + CanonPath p("/baz//quux/", p0); + ASSERT_EQ(p.abs(), "/baz/quux"); + ASSERT_EQ(p.rel(), "baz/quux"); + ASSERT_EQ(*p.baseName(), "quux"); + ASSERT_EQ(*p.dirOf(), "/baz"); + } + { + CanonPath p("baz//quux/", p0); + ASSERT_EQ(p.abs(), "/foo/bar/baz/quux"); + ASSERT_EQ(p.rel(), "foo/bar/baz/quux"); + ASSERT_EQ(*p.baseName(), "quux"); + ASSERT_EQ(*p.dirOf(), "/foo/bar/baz"); + } + } + TEST(CanonPath, pop) { CanonPath p("foo/bar/x"); ASSERT_EQ(p.abs(), "/foo/bar/x"); @@ -80,29 +98,29 @@ namespace nix { { CanonPath p1("a//foo/bar//"); CanonPath p2("xyzzy/bla"); - ASSERT_EQ((p1 + p2).abs(), "/a/foo/bar/xyzzy/bla"); + ASSERT_EQ((p1 / p2).abs(), "/a/foo/bar/xyzzy/bla"); } { CanonPath p1("/"); CanonPath p2("/a/b"); - ASSERT_EQ((p1 + p2).abs(), "/a/b"); + ASSERT_EQ((p1 / p2).abs(), "/a/b"); } { CanonPath p1("/a/b"); CanonPath p2("/"); - ASSERT_EQ((p1 + p2).abs(), "/a/b"); + ASSERT_EQ((p1 / p2).abs(), "/a/b"); } { CanonPath p("/foo/bar"); - ASSERT_EQ((p + "x").abs(), "/foo/bar/x"); + ASSERT_EQ((p / "x").abs(), "/foo/bar/x"); } { CanonPath p("/"); - ASSERT_EQ((p + "foo" + "bar").abs(), "/foo/bar"); + ASSERT_EQ((p / "foo" / "bar").abs(), "/foo/bar"); } } diff --git a/tests/unit/libutil/file-content-address.cc b/tests/unit/libutil/file-content-address.cc new file mode 100644 index 000000000..294e39806 --- /dev/null +++ b/tests/unit/libutil/file-content-address.cc @@ -0,0 +1,61 @@ +#include + +#include "file-content-address.hh" + +namespace nix { + +/* ---------------------------------------------------------------------------- + * parseFileSerialisationMethod, renderFileSerialisationMethod + * --------------------------------------------------------------------------*/ + +TEST(FileSerialisationMethod, testRoundTripPrintParse_1) { + for (const FileSerialisationMethod fim : { + FileSerialisationMethod::Flat, + FileSerialisationMethod::Recursive, + }) { + EXPECT_EQ(parseFileSerialisationMethod(renderFileSerialisationMethod(fim)), fim); + } +} + +TEST(FileSerialisationMethod, testRoundTripPrintParse_2) { + for (const std::string_view fimS : { + "flat", + "nar", + }) { + EXPECT_EQ(renderFileSerialisationMethod(parseFileSerialisationMethod(fimS)), fimS); + } +} + +TEST(FileSerialisationMethod, testParseFileSerialisationMethodOptException) { + EXPECT_THROW(parseFileSerialisationMethod("narwhal"), UsageError); +} + +/* ---------------------------------------------------------------------------- + * parseFileIngestionMethod, renderFileIngestionMethod + * --------------------------------------------------------------------------*/ + +TEST(FileIngestionMethod, testRoundTripPrintParse_1) { + for (const FileIngestionMethod fim : { + FileIngestionMethod::Flat, + FileIngestionMethod::Recursive, + FileIngestionMethod::Git, + }) { + EXPECT_EQ(parseFileIngestionMethod(renderFileIngestionMethod(fim)), fim); + } +} + +TEST(FileIngestionMethod, testRoundTripPrintParse_2) { + for (const std::string_view fimS : { + "flat", + "nar", + "git", + }) { + EXPECT_EQ(renderFileIngestionMethod(parseFileIngestionMethod(fimS)), fimS); + } +} + +TEST(FileIngestionMethod, testParseFileIngestionMethodOptException) { + EXPECT_THROW(parseFileIngestionMethod("narwhal"), UsageError); +} + +} diff --git a/tests/unit/libutil/logging.cc b/tests/unit/libutil/logging.cc index 8950a26d4..1d7304f05 100644 --- a/tests/unit/libutil/logging.cc +++ b/tests/unit/libutil/logging.cc @@ -42,7 +42,7 @@ namespace nix { makeJSONLogger(*logger)->logEI({ .name = "error name", - .msg = hintfmt("this hint has %1% templated %2%!!", + .msg = HintFmt("this hint has %1% templated %2%!!", "yellow", "values"), .errPos = Pos(foFile, problem_file, 02, 13) @@ -62,7 +62,7 @@ namespace nix { throw TestError(e.info()); } catch (Error &e) { ErrorInfo ei = e.info(); - ei.msg = hintfmt("%s; subsequent error message.", normaltxt(e.info().msg.str())); + ei.msg = HintFmt("%s; subsequent error message.", Uncolored(e.info().msg.str())); testing::internal::CaptureStderr(); logger->logEI(ei); @@ -176,7 +176,7 @@ namespace nix { logError({ .name = "error name", - .msg = hintfmt("this hint has %1% templated %2%!!", + .msg = HintFmt("this hint has %1% templated %2%!!", "yellow", "values"), .errPos = Pos(foString, problem_file, 02, 13), @@ -193,7 +193,7 @@ namespace nix { logError({ .name = "error name", - .msg = hintfmt("this hint has %1% templated %2%!!", + .msg = HintFmt("this hint has %1% templated %2%!!", "yellow", "values"), .errPos = Pos(foFile, problem_file, 02, 13) @@ -208,7 +208,7 @@ namespace nix { logError({ .name = "error name", - .msg = hintfmt("hint %1%", "only"), + .msg = HintFmt("hint %1%", "only"), }); auto str = testing::internal::GetCapturedStderr(); @@ -225,7 +225,7 @@ namespace nix { logWarning({ .name = "name", - .msg = hintfmt("there was a %1%", "warning"), + .msg = HintFmt("there was a %1%", "warning"), }); auto str = testing::internal::GetCapturedStderr(); @@ -241,7 +241,7 @@ namespace nix { logWarning({ .name = "warning name", - .msg = hintfmt("this hint has %1% templated %2%!!", + .msg = HintFmt("this hint has %1% templated %2%!!", "yellow", "values"), .errPos = Pos(foStdin, problem_file, 2, 13), @@ -264,7 +264,7 @@ namespace nix { auto e = AssertionError(ErrorInfo { .name = "wat", - .msg = hintfmt("it has been %1% days since our last error", "zero"), + .msg = HintFmt("it has been %1% days since our last error", "zero"), .errPos = Pos(foString, problem_file, 2, 13), }); @@ -290,7 +290,7 @@ namespace nix { auto e = AssertionError(ErrorInfo { .name = "wat", - .msg = hintfmt("it has been %1% days since our last error", "zero"), + .msg = HintFmt("it has been %1% days since our last error", "zero"), .errPos = Pos(foString, problem_file, 2, 13), }); @@ -310,39 +310,39 @@ namespace nix { /* ---------------------------------------------------------------------------- - * hintfmt + * HintFmt * --------------------------------------------------------------------------*/ - TEST(hintfmt, percentStringWithoutArgs) { + TEST(HintFmt, percentStringWithoutArgs) { const char *teststr = "this is 100%s correct!"; ASSERT_STREQ( - hintfmt(teststr).str().c_str(), + HintFmt(teststr).str().c_str(), teststr); } - TEST(hintfmt, fmtToHintfmt) { + TEST(HintFmt, fmtToHintfmt) { ASSERT_STREQ( - hintfmt(fmt("the color of this this text is %1%", "not yellow")).str().c_str(), + HintFmt(fmt("the color of this this text is %1%", "not yellow")).str().c_str(), "the color of this this text is not yellow"); } - TEST(hintfmt, tooFewArguments) { + TEST(HintFmt, tooFewArguments) { ASSERT_STREQ( - hintfmt("only one arg %1% %2%", "fulfilled").str().c_str(), + HintFmt("only one arg %1% %2%", "fulfilled").str().c_str(), "only one arg " ANSI_WARNING "fulfilled" ANSI_NORMAL " "); } - TEST(hintfmt, tooManyArguments) { + TEST(HintFmt, tooManyArguments) { ASSERT_STREQ( - hintfmt("what about this %1% %2%", "%3%", "one", "two").str().c_str(), + HintFmt("what about this %1% %2%", "%3%", "one", "two").str().c_str(), "what about this " ANSI_WARNING "%3%" ANSI_NORMAL " " ANSI_YELLOW "one" ANSI_NORMAL); } diff --git a/tests/unit/libutil/tests.cc b/tests/unit/libutil/tests.cc index 568f03f70..4406fd184 100644 --- a/tests/unit/libutil/tests.cc +++ b/tests/unit/libutil/tests.cc @@ -9,6 +9,14 @@ #include +#ifdef _WIN32 +# define FS_SEP "\\" +# define FS_ROOT "C:" FS_SEP // Need a mounted one, C drive is likely +#else +# define FS_SEP "/" +# define FS_ROOT FS_SEP +#endif + namespace nix { /* ----------- tests for util.hh ------------------------------------------------*/ @@ -18,9 +26,9 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(absPath, doesntChangeRoot) { - auto p = absPath("/"); + auto p = absPath(FS_ROOT); - ASSERT_EQ(p, "/"); + ASSERT_EQ(p, FS_ROOT); } @@ -53,11 +61,11 @@ namespace nix { TEST(absPath, pathIsCanonicalised) { - auto path = "/some/path/with/trailing/dot/."; + auto path = FS_ROOT "some/path/with/trailing/dot/."; auto p1 = absPath(path); auto p2 = absPath(p1); - ASSERT_EQ(p1, "/some/path/with/trailing/dot"); + ASSERT_EQ(p1, FS_ROOT "some" FS_SEP "path" FS_SEP "with" FS_SEP "trailing" FS_SEP "dot"); ASSERT_EQ(p1, p2); } @@ -66,24 +74,24 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(canonPath, removesTrailingSlashes) { - auto path = "/this/is/a/path//"; + auto path = FS_ROOT "this/is/a/path//"; auto p = canonPath(path); - ASSERT_EQ(p, "/this/is/a/path"); + ASSERT_EQ(p, FS_ROOT "this" FS_SEP "is" FS_SEP "a" FS_SEP "path"); } TEST(canonPath, removesDots) { - auto path = "/this/./is/a/path/./"; + auto path = FS_ROOT "this/./is/a/path/./"; auto p = canonPath(path); - ASSERT_EQ(p, "/this/is/a/path"); + ASSERT_EQ(p, FS_ROOT "this" FS_SEP "is" FS_SEP "a" FS_SEP "path"); } TEST(canonPath, removesDots2) { - auto path = "/this/a/../is/a////path/foo/.."; + auto path = FS_ROOT "this/a/../is/a////path/foo/.."; auto p = canonPath(path); - ASSERT_EQ(p, "/this/is/a/path"); + ASSERT_EQ(p, FS_ROOT "this" FS_SEP "is" FS_SEP "a" FS_SEP "path"); } TEST(canonPath, requiresAbsolutePath) { @@ -197,7 +205,7 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(pathExists, rootExists) { - ASSERT_TRUE(pathExists("/")); + ASSERT_TRUE(pathExists(FS_ROOT)); } TEST(pathExists, cwdExists) {