diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml index 5f949ddc5..d83cb4f18 100644 --- a/.github/workflows/labels.yml +++ b/.github/workflows/labels.yml @@ -21,4 +21,4 @@ jobs: - uses: actions/labeler@v4 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - sync-labels: true + sync-labels: false diff --git a/.gitignore b/.gitignore index 969194650..93a9ff9ae 100644 --- a/.gitignore +++ b/.gitignore @@ -95,6 +95,7 @@ perl/Makefile.config # /tests/lang/ /tests/lang/*.out /tests/lang/*.out.xml +/tests/lang/*.err /tests/lang/*.ast /perl/lib/Nix/Config.pm diff --git a/.version b/.version index d76bd2ba3..cf8690732 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.17.0 +2.18.0 diff --git a/Makefile b/Makefile index c6220482a..31b54b93d 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,8 @@ makefiles += \ src/libstore/tests/local.mk \ src/libexpr/tests/local.mk \ tests/local.mk \ + tests/ca/local.mk \ + tests/dyn-drv/local.mk \ tests/test-libstoreconsumer/local.mk \ tests/plugins/local.mk else diff --git a/configure.ac b/configure.ac index bb3f92e4d..6d78237f0 100644 --- a/configure.ac +++ b/configure.ac @@ -5,7 +5,14 @@ AC_CONFIG_AUX_DIR(config) AC_PROG_SED -# Construct a Nix system name (like "i686-linux"). +# Construct a Nix system name (like "i686-linux"): +# https://www.gnu.org/software/autoconf/manual/html_node/Canonicalizing.html#index-AC_005fCANONICAL_005fHOST-1 +# The inital value is produced by the `config/config.guess` script: +# upstream: https://git.savannah.gnu.org/cgit/config.git/tree/config.guess +# It has the following form, which is not documented anywhere: +# --[][-] +# If `./configure` is passed any of the `--host`, `--build`, `--target` options, the value comes from `config/config.sub` instead: +# upstream: https://git.savannah.gnu.org/cgit/config.git/tree/config.sub AC_CANONICAL_HOST AC_MSG_CHECKING([for the canonical Nix system name]) diff --git a/doc/manual/generate-builtin-constants.nix b/doc/manual/generate-builtin-constants.nix index 3fc1fae42..8af80a02c 100644 --- a/doc/manual/generate-builtin-constants.nix +++ b/doc/manual/generate-builtin-constants.nix @@ -10,12 +10,14 @@ let type' = optionalString (type != null) " (${type})"; impureNotice = optionalString impure-only '' - Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). + > **Note** + > + > Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). ''; in squash '' -
- ${name}${type'} +
+ ${name}${type'}
diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index fb34898f3..65eec42d0 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -137,12 +137,29 @@ let storeDocs = let - showStore = name: { settings, doc }: - '' + showStore = name: { settings, doc, experimentalFeature }: + let + experimentalFeatureNote = optionalString (experimentalFeature != null) '' + > **Warning** + > This store is part of an + > [experimental feature](@docroot@/contributing/experimental-features.md). + + To use this store, you need to make sure the corresponding experimental feature, + [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}), + is enabled. + For example, include the following in [`nix.conf`](#): + + ``` + extra-experimental-features = ${experimentalFeature} + ``` + ''; + in '' ## ${name} ${doc} + ${experimentalFeatureNote} + **Settings**: ${showSettings { useAnchors = false; } settings} diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index dcdb5d6e9..b43622ed6 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -281,7 +281,7 @@ const redirects = { "chap-introduction": "introduction.html", "ch-basic-package-mgmt": "package-management/basic-package-mgmt.html", "ssec-binary-cache-substituter": "package-management/binary-cache-substituter.html", - "sec-channels": "package-management/channels.html", + "sec-channels": "command-ref/nix-channel.html", "ssec-copy-closure": "package-management/copy-closure.html", "sec-garbage-collection": "package-management/garbage-collection.html", "ssec-gc-roots": "package-management/garbage-collector-roots.html", diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 1bd8fa774..6c599abcf 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -21,7 +21,6 @@ - [Profiles](package-management/profiles.md) - [Garbage Collection](package-management/garbage-collection.md) - [Garbage Collector Roots](package-management/garbage-collector-roots.md) - - [Channels](package-management/channels.md) - [Sharing Packages Between Machines](package-management/sharing-packages.md) - [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md) - [Copying Closures via SSH](package-management/copy-closure.md) @@ -110,6 +109,7 @@ - [C++ style guide](contributing/cxx.md) - [Release Notes](release-notes/release-notes.md) - [Release X.Y (202?-??-??)](release-notes/rl-next.md) + - [Release 2.17 (2023-07-24)](release-notes/rl-2.17.md) - [Release 2.16 (2023-05-31)](release-notes/rl-2.16.md) - [Release 2.15 (2023-04-11)](release-notes/rl-2.15.md) - [Release 2.14 (2023-02-28)](release-notes/rl-2.14.md) diff --git a/doc/manual/src/command-ref/nix-channel.md b/doc/manual/src/command-ref/nix-channel.md index 025f758e7..cebbc7b00 100644 --- a/doc/manual/src/command-ref/nix-channel.md +++ b/doc/manual/src/command-ref/nix-channel.md @@ -8,36 +8,46 @@ # Description -A Nix channel is a mechanism that allows you to automatically stay -up-to-date with a set of pre-built Nix expressions. A Nix channel is -just a URL that points to a place containing a set of Nix expressions. +Channels are a mechanism for referencing remote Nix expressions and conveniently retrieving their latest version. -To see the list of official NixOS channels, visit -. +The moving parts of channels are: +- The official channels listed at +- The user-specific list of [subscribed channels](#subscribed-channels) +- The [downloaded channel contents](#channels) +- The [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path), set with the [`-I` option](#opt-i) or the [`NIX_PATH` environment variable](#env-NIX_PATH) + +> **Note** +> +> The state of a subscribed channel is external to the Nix expressions relying on it. +> This may limit reproducibility. +> +> Dependencies on other Nix expressions can be declared explicitly with: +> - [`fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl), [`fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball), or [`fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) in Nix expressions +> - the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I) in command line invocations This command has the following operations: - `--add` *url* \[*name*\]\ - Adds a channel named *name* with URL *url* to the list of subscribed - channels. If *name* is omitted, it defaults to the last component of - *url*, with the suffixes `-stable` or `-unstable` removed. + Add a channel *name* located at *url* to the list of subscribed channels. + If *name* is omitted, default to the last component of *url*, with the suffixes `-stable` or `-unstable` removed. + + > **Note** + > + > `--add` does not automatically perform an update. + > Use `--update` explicitly. A channel URL must point to a directory containing a file `nixexprs.tar.gz`. At the top level, that tarball must contain a single directory with a `default.nix` file that serves as the channel’s entry point. - `--remove` *name*\ - Removes the channel named *name* from the list of subscribed - channels. + Remove the channel *name* from the list of subscribed channels. - `--list`\ - Prints the names and URLs of all subscribed channels on standard - output. + Print the names and URLs of all subscribed channels on standard output. - `--update` \[*names*…\]\ - Downloads the Nix expressions of all subscribed channels (or only - those included in *names* if specified) and makes them the default - for `nix-env` operations (by symlinking them from the directory - `~/.nix-defexpr`). + Download the Nix expressions of subscribed channels and create a new generation. + Update all channels if none is specified, and only those included in *names* otherwise. - `--list-generations`\ Prints a list of all the current existing generations for the @@ -49,13 +59,8 @@ This command has the following operations: ``` - `--rollback` \[*generation*\]\ - Reverts the previous call to `nix-channel - --update`. Optionally, you can specify a specific channel generation - number to restore. - -Note that `--add` does not automatically perform an update. - -The list of subscribed channels is stored in `~/.nix-channels`. + Revert channels to the state before the last call to `nix-channel --update`. + Optionally, you can specify a specific channel *generation* number to restore. {{#include ./opt-common.md}} @@ -69,23 +74,33 @@ The list of subscribed channels is stored in `~/.nix-channels`. # Examples -To subscribe to the Nixpkgs channel and install the GNU Hello package: +Subscribe to the Nixpkgs channel and run `hello` from the GNU Hello package: ```console $ nix-channel --add https://nixos.org/channels/nixpkgs-unstable +$ nix-channel --list +nixpkgs https://nixos.org/channels/nixpkgs $ nix-channel --update -$ nix-env --install --attr nixpkgs.hello +$ nix-shell -p hello --run hello +hello ``` -You can revert channel updates using `--rollback`: +Revert channel updates using `--rollback`: ```console -$ nix-instantiate --eval --expr '(import {}).lib.version' -"14.04.527.0e935f1" +$ nix-instantiate --eval '' --attr lib.version +"22.11pre296212.530a53dcbc9" $ nix-channel --rollback switching from generation 483 to 482 -$ nix-instantiate --eval --expr '(import {}).lib.version' -"14.04.526.dbadfad" +$ nix-instantiate --eval '' --attr lib.version +"22.11pre281526.d0419badfad" +``` + +Remove a channel: + +```console +$ nix-channel --remove nixpkgs +$ nix-channel --list ``` diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 7b2440971..4b0a3a3e5 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -110,41 +110,72 @@ You can also build Nix for one of the [supported platforms](#platforms). ## Platforms -As specified in [`flake.nix`], Nix can be built for various platforms: - -- `aarch64-linux` -- `i686-linux` -- `x86_64-darwin` -- `x86_64-linux` +Nix can be built for various platforms, as specified in [`flake.nix`]: [`flake.nix`]: https://github.com/nixos/nix/blob/master/flake.nix +- `x86_64-linux` +- `x86_64-darwin` +- `i686-linux` +- `aarch64-linux` +- `aarch64-darwin` +- `armv6l-linux` +- `armv7l-linux` + In order to build Nix for a different platform than the one you're currently -on, you need to have some way for your system Nix to build code for that -platform. Common solutions include [remote builders] and [binfmt emulation] +on, you need a way for your current Nix installation to build code for that +platform. Common solutions include [remote builders] and [binary format emulation] (only supported on NixOS). [remote builders]: ../advanced-topics/distributed-builds.md -[binfmt emulation]: https://nixos.org/manual/nixos/stable/options.html#opt-boot.binfmt.emulatedSystems +[binary format emulation]: https://nixos.org/manual/nixos/stable/options.html#opt-boot.binfmt.emulatedSystems -These solutions let Nix perform builds as if you're on the native platform, so -executing the build is as simple as - -```console -$ nix build .#packages.aarch64-linux.default -``` - -for flake-enabled Nix, or +Given such a setup, executing the build only requires selecting the respective attribute. +For example, to compile for `aarch64-linux`: ```console $ nix-build --attr packages.aarch64-linux.default ``` -for classic Nix. +or for Nix with the [`flakes`] and [`nix-command`] experimental features enabled: -You can use any of the other supported platforms in place of `aarch64-linux`. +```console +$ nix build .#packages.aarch64-linux.default +``` -Cross-compiled builds are available for ARMv6 and ARMv7, and Nix on unsupported platforms can be bootstrapped by adding more `crossSystems` in `flake.nix`. +Cross-compiled builds are available for ARMv6 (`armv6l-linux`) and ARMv7 (`armv7l-linux`). +Add more [system types](#system-type) to `crossSystems` in `flake.nix` to bootstrap Nix on unsupported platforms. + +## System type + +Nix uses a string with he following format to identify the *system type* or *platform* it runs on: + +``` +-[-] +``` + +It is set when Nix is compiled for the given system, and based on the output of [`config.guess`](https://github.com/nixos/nix/blob/master/config/config.guess) ([upstream](https://git.savannah.gnu.org/cgit/config.git/tree/config.guess)): + +``` +--[][-] +``` + +When Nix is built such that `./configure` is passed any of the `--host`, `--build`, `--target` options, the value is based on the output of [`config.sub`](https://github.com/nixos/nix/blob/master/config/config.sub) ([upstream](https://git.savannah.gnu.org/cgit/config.git/tree/config.sub)): + +``` +-[-]- +``` + +For historic reasons and backward-compatibility, some CPU and OS identifiers are translated from the GNU Autotools naming convention in [`configure.ac`](https://github.com/nixos/nix/blob/master/configure.ac) as follows: + +| `config.guess` | Nix | +|----------------------------|---------------------| +| `amd64` | `x86_64` | +| `i*86` | `i686` | +| `arm6` | `arm6l` | +| `arm7` | `arm7l` | +| `linux-gnu*` | `linux` | +| `linux-musl*` | `linux` | ## Compilation environments diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index e5f80a928..c3c82e3c0 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -14,6 +14,8 @@ You can run the whole testsuite with `make check`, or the tests for a specific c The functional tests reside under the `tests` directory and are listed in `tests/local.mk`. Each test is a bash script. +### Running the whole test suite + The whole test suite can be run with: ```shell-session @@ -23,6 +25,33 @@ ran test tests/bar.sh... [PASS] ... ``` +### Grouping tests + +Sometimes it is useful to group related tests so they can be easily run together without running the entire test suite. +Each test group is in a subdirectory of `tests`. +For example, `tests/ca/local.mk` defines a `ca` test group for content-addressed derivation outputs. + +That test group can be run like this: + +```shell-session +$ make ca.test-group -j50 +ran test tests/ca/nix-run.sh... [PASS] +ran test tests/ca/import-derivation.sh... [PASS] +... +``` + +The test group is defined in Make like this: +```makefile +$(test-group-name)-tests := \ + $(d)/test0.sh \ + $(d)/test1.sh \ + ... + +install-tests-groups += $(test-group-name) +``` + +### Running individual tests + Individual tests can be run with `make`: ```shell-session @@ -86,6 +115,31 @@ GNU gdb (GDB) 12.1 One can debug the Nix invocation in all the usual ways. For example, enter `run` to start the Nix invocation. +### Characterization testing + +Occasionally, Nix utilizes a technique called [Characterization Testing](https://en.wikipedia.org/wiki/Characterization_test) as part of the functional tests. +This technique is to include the exact output/behavior of a former version of Nix in a test in order to check that Nix continues to produce the same behavior going forward. + +For example, this technique is used for the language tests, to check both the printed final value if evaluation was successful, and any errors and warnings encountered. + +It is frequently useful to regenerate the expected output. +To do that, rerun the failed test with `_NIX_TEST_ACCEPT=1`. +(At least, this is the convention we've used for `tests/lang.sh`. +If we add more characterization testing we should always strive to be consistent.) + +An interesting situation to document is the case when these tests are "overfitted". +The language tests are, again, an example of this. +The expected successful output of evaluation is supposed to be highly stable – we do not intend to make breaking changes to (the stable parts of) the Nix language. +However, the errors and warnings during evaluation (successful or not) are not stable in this way. +We are free to change how they are displayed at any time. + +It may be surprising that we would test non-normative behavior like diagnostic outputs. +Diagnostic outputs are indeed not a stable interface, but they still are important to users. +By recording the expected output, the test suite guards against accidental changes, and ensure the *result* (not just the code that implements it) of the diagnostic code paths are under code review. +Regressions are caught, and improvements always show up in code review. + +To ensure that characterization testing doesn't make it harder to intentionally change these interfaces, there always must be an easy way to regenerate the expected output, as we do with `_NIX_TEST_ACCEPT=1`. + ## Integration tests The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute. diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 307971434..5e8aaeba0 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -320,16 +320,6 @@ Derivations can declare some infrequently used optional attributes. ``` - [`unsafeDiscardReferences`]{#adv-attr-unsafeDiscardReferences}\ - > **Warning** - > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md). - > - > To use this attribute, you must enable the - > [`discard-references`](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) experimental feature. - > For example, in [nix.conf](../command-ref/conf-file.md) you could add: - > - > ``` - > extra-experimental-features = discard-references - > ``` When using [structured attributes](#adv-attr-structuredAttrs), the attribute `unsafeDiscardReferences` is an attribute set with a boolean value for each output name. diff --git a/doc/manual/src/language/constructs.md b/doc/manual/src/language/constructs.md index c53eb8889..a3590f55d 100644 --- a/doc/manual/src/language/constructs.md +++ b/doc/manual/src/language/constructs.md @@ -92,10 +92,10 @@ In this fragment from `all-packages.nix`, ```nix graphviz = (import ../tools/graphics/graphviz) { inherit fetchurl stdenv libpng libjpeg expat x11 yacc; - inherit (xlibs) libXaw; + inherit (xorg) libXaw; }; -xlibs = { +xorg = { libX11 = ...; libXaw = ...; ... @@ -109,7 +109,7 @@ libjpg = ...; the set used in the function call to the function defined in `../tools/graphics/graphviz` inherits a number of variables from the surrounding scope (`fetchurl` ... `yacc`), but also inherits `libXaw` -(the X Athena Widgets) from the `xlibs` (X11 client-side libraries) set. +(the X Athena Widgets) from the `xorg` set. Summarizing the fragment @@ -208,30 +208,41 @@ three kinds of patterns: ```nix { x, y, z, ... } @ args: z + y + x + args.a ``` - - Here `args` is bound to the entire argument, which is further - matched against the pattern `{ x, y, z, - ... }`. `@`-pattern makes mainly sense with an ellipsis(`...`) as + + Here `args` is bound to the argument *as passed*, which is further + matched against the pattern `{ x, y, z, ... }`. + The `@`-pattern makes mainly sense with an ellipsis(`...`) as you can access attribute names as `a`, using `args.a`, which was given as an additional attribute to the function. - + > **Warning** - > - > The `args@` expression is bound to the argument passed to the - > function which means that attributes with defaults that aren't - > explicitly specified in the function call won't cause an - > evaluation error, but won't exist in `args`. - > + > + > `args@` binds the name `args` to the attribute set that is passed to the function. + > In particular, `args` does *not* include any default values specified with `?` in the function's set pattern. + > > For instance - > + > > ```nix > let - > function = args@{ a ? 23, ... }: args; + > f = args@{ a ? 23, ... }: [ a args ]; > in - > function {} - > ```` - > - > will evaluate to an empty attribute set. + > f {} + > ``` + > + > is equivalent to + > + > ```nix + > let + > f = args @ { ... }: [ (args.a or 23) args ]; + > in + > f {} + > ``` + > + > and both expressions will evaluate to: + > + > ```nix + > [ 23 {} ] + > ``` Note that functions do not have names. If you want to give them a name, you can bind them to an attribute, e.g., diff --git a/doc/manual/src/package-management/basic-package-mgmt.md b/doc/manual/src/package-management/basic-package-mgmt.md index 6b86e763e..07b92fb76 100644 --- a/doc/manual/src/package-management/basic-package-mgmt.md +++ b/doc/manual/src/package-management/basic-package-mgmt.md @@ -25,7 +25,7 @@ or completely new ones.) You can manually download the latest version of Nixpkgs from . However, it’s much more -convenient to use the Nixpkgs [*channel*](channels.md), since it makes +convenient to use the Nixpkgs [*channel*](../command-ref/nix-channel.md), since it makes it easy to stay up to date with new versions of Nixpkgs. Nixpkgs is automatically added to your list of “subscribed” channels when you install Nix. If this is not the case for some reason, you can add it diff --git a/doc/manual/src/package-management/channels.md b/doc/manual/src/package-management/channels.md deleted file mode 100644 index 8e4da180b..000000000 --- a/doc/manual/src/package-management/channels.md +++ /dev/null @@ -1,50 +0,0 @@ -# Channels - -If you want to stay up to date with a set of packages, it’s not very -convenient to manually download the latest set of Nix expressions for -those packages and upgrade using `nix-env`. Fortunately, there’s a -better way: *Nix channels*. - -A Nix channel is just a URL that points to a place that contains a set -of Nix expressions and a manifest. Using the command -[`nix-channel`](../command-ref/nix-channel.md) you can automatically -stay up to date with whatever is available at that URL. - -To see the list of official NixOS channels, visit -. - -You can “subscribe” to a channel using `nix-channel --add`, e.g., - -```console -$ nix-channel --add https://nixos.org/channels/nixpkgs-unstable -``` - -subscribes you to a channel that always contains that latest version of -the Nix Packages collection. (Subscribing really just means that the URL -is added to the file `~/.nix-channels`, where it is read by subsequent -calls to `nix-channel ---update`.) You can “unsubscribe” using `nix-channel ---remove`: - -```console -$ nix-channel --remove nixpkgs -``` - -To obtain the latest Nix expressions available in a channel, do - -```console -$ nix-channel --update -``` - -This downloads and unpacks the Nix expressions in every channel -(downloaded from `url/nixexprs.tar.bz2`). It also makes the union of -each channel’s Nix expressions available by default to `nix-env` -operations (via the symlink `~/.nix-defexpr/channels`). Consequently, -you can then say - -```console -$ nix-env --upgrade -``` - -to upgrade all packages in your profile to the latest versions available -in the subscribed channels. diff --git a/doc/manual/src/release-notes/rl-2.17.md b/doc/manual/src/release-notes/rl-2.17.md new file mode 100644 index 000000000..0b861aecc --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.17.md @@ -0,0 +1,42 @@ +# Release 2.17 (2023-07-24) + +* [`nix-channel`](../command-ref/nix-channel.md) now supports a `--list-generations` subcommand. + +* The function [`builtins.fetchClosure`](../language/builtins.md#builtins-fetchClosure) can now fetch input-addressed paths in [pure evaluation mode](../command-ref/conf-file.md#conf-pure-eval), as those are not impure. + +* Nix now allows unprivileged/[`allowed-users`](../command-ref/conf-file.md#conf-allowed-users) to sign paths. + Previously, only [`trusted-users`](../command-ref/conf-file.md#conf-trusted-users) users could sign paths. + +* Nested dynamic attributes are now merged correctly by the parser. For example: + + ```nix + { + nested = { + foo = 1; + }; + nested = { + ${"ba" + "r"} = 2; + }; + } + ``` + + This used to silently discard `nested.bar`, but now behaves as one would expect and evaluates to: + + ```nix + { nested = { bar = 2; foo = 1; }; } + ``` + + Note that the feature of merging multiple *full declarations* of attribute sets like `nested` in the example is of questionable value. + It allows writing expressions that are very hard to read, for instance when there are many lines of code between two declarations of the same attribute. + This has been around for a long time and is therefore supported for backwards compatibility, but should not be relied upon. + + Instead, consider using the *nested attribute path* syntax: + + ```nix + { + nested.foo = 1; + nested.${"ba" + "r"} = 2; + } + ``` + +* Tarball flakes can now redirect to an "immutable" URL that will be recorded in lock files. This allows the use of "mutable" tarball URLs like `https://example.org/hello/latest.tar.gz` in flakes. See the [tarball fetcher](../protocols/tarball-fetcher.md) for details. diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 139d07188..be922f95a 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,8 +1,21 @@ # Release X.Y (202?-??-??) -- [`nix-channel`](../command-ref/nix-channel.md) now supports a `--list-generations` subcommand +- Two new builtin functions, + [`builtins.parseFlakeRef`](@docroot@/language/builtins.md#builtins-parseFlakeRef) + and + [`builtins.flakeRefToString`](@docroot@/language/builtins.md#builtins-flakeRefToString), + have been added. + These functions are useful for converting between flake references encoded as attribute sets and URLs. -* The function [`builtins.fetchClosure`](../language/builtins.md#builtins-fetchClosure) can now fetch input-addressed paths in [pure evaluation mode](../command-ref/conf-file.md#conf-pure-eval), as those are not impure. +- [`builtins.toJSON`](@docroot@/language/builtins.md#builtins-parseFlakeRef) now prints [--show-trace](@docroot@/command-ref/conf-file.html#conf-show-trace) items for the path in which it finds an evaluation error. -- Nix now allows unprivileged/[`allowed-users`](../command-ref/conf-file.md#conf-allowed-users) to sign paths. - Previously, only [`trusted-users`](../command-ref/conf-file.md#conf-trusted-users) users could sign paths. +- Error messages regarding malformed input to [`derivation add`](@docroot@/command-ref/new-cli/nix3-derivation-add.md) are now clearer and more detailed. + +- The `discard-references` feature has been stabilized. + This means that the + [unsafeDiscardReferences](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) + attribute is no longer guarded by an experimental flag and can be used + freely. + +- The JSON output for derived paths with are store paths is now a string, not an object with a single `path` field. + This only affects `nix-build --json` when "building" non-derivation things like fetched sources, which is a no-op. diff --git a/mk/lib.mk b/mk/lib.mk index 34fa624d8..e86a7f1a4 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -10,6 +10,7 @@ bin-scripts := noinst-scripts := man-pages := install-tests := +install-tests-groups := ifdef HOST_OS HOST_KERNEL = $(firstword $(subst -, ,$(HOST_OS))) @@ -121,7 +122,16 @@ $(foreach script, $(bin-scripts), $(eval $(call install-program-in,$(script),$(b $(foreach script, $(bin-scripts), $(eval programs-list += $(script))) $(foreach script, $(noinst-scripts), $(eval programs-list += $(script))) $(foreach template, $(template-files), $(eval $(call instantiate-template,$(template)))) -$(foreach test, $(install-tests), $(eval $(call run-install-test,$(test)))) +$(foreach test, $(install-tests), \ + $(eval $(call run-install-test,$(test))) \ + $(eval installcheck: $(test).test)) +$(foreach test-group, $(install-tests-groups), \ + $(eval $(call run-install-test-group,$(test-group))) \ + $(eval installcheck: $(test-group).test-group) \ + $(foreach test, $($(test-group)-tests), \ + $(eval $(call run-install-test,$(test))) \ + $(eval $(test-group).test-group: $(test).test))) + $(foreach file, $(man-pages), $(eval $(call install-data-in, $(file), $(mandir)/man$(patsubst .%,%,$(suffix $(file)))))) @@ -151,6 +161,14 @@ ifdef libs-list @echo "The following libraries can be built:" @echo "" @for i in $(libs-list); do echo " $$i"; done +endif +ifdef install-tests-groups + @echo "" + @echo "The following groups of functional tests can be run:" + @echo "" + @for i in $(install-tests-groups); do echo " $$i.test-group"; done + @echo "" + @echo "(installcheck includes tests in test groups too.)" endif @echo "" @echo "The following variables control the build:" diff --git a/mk/tests.mk b/mk/tests.mk index 3ebbd86e3..ec8128bdf 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -4,8 +4,6 @@ test-deps = define run-install-test - installcheck: $1.test - .PHONY: $1.test $1.test: $1 $(test-deps) @env BASH=$(bash) $(bash) mk/run-test.sh $1 < /dev/null @@ -16,6 +14,12 @@ define run-install-test endef +define run-install-test-group + + .PHONY: $1.test-group + +endef + .PHONY: check installcheck print-top-help += \ diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 41ecbbeb4..c38ea2d2b 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -294,10 +294,8 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) auto h = Hash::parseAny(hash, parseHashType(algo)); auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; auto path = store()->makeFixedOutputPath(name, FixedOutputInfo { - .hash = { - .method = method, - .hash = h, - }, + .method = method, + .hash = h, .references = {}, }); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 2fb17d06f..c1f03a8ef 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -322,7 +322,12 @@ connected: throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg); } else { copyClosure(*store, *sshStore, StorePathSet {*drvPath}, NoRepair, NoCheckSigs, substitute); - auto res = sshStore->buildPathsWithResults({ DerivedPath::Built { *drvPath, OutputsSpec::All {} } }); + auto res = sshStore->buildPathsWithResults({ + DerivedPath::Built { + .drvPath = makeConstantStorePathRef(*drvPath), + .outputs = OutputsSpec::All {}, + } + }); // One path to build should produce exactly one build result assert(res.size() == 1); optResult = std::move(res[0]); diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc new file mode 100644 index 000000000..c6cc7fa9c --- /dev/null +++ b/src/libcmd/built-path.cc @@ -0,0 +1,127 @@ +#include "built-path.hh" +#include "derivations.hh" +#include "store-api.hh" + +#include + +#include + +namespace nix { + +#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \ + bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \ + { \ + const MY_TYPE* me = this; \ + auto fields1 = std::make_tuple(*me->drvPath, me->FIELD); \ + me = &other; \ + auto fields2 = std::make_tuple(*me->drvPath, me->FIELD); \ + return fields1 COMPARATOR fields2; \ + } +#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <) + +#define FIELD_TYPE std::pair +CMP(SingleBuiltPath, SingleBuiltPathBuilt, output) +#undef FIELD_TYPE + +#define FIELD_TYPE std::map +CMP(SingleBuiltPath, BuiltPathBuilt, outputs) +#undef FIELD_TYPE + +#undef CMP +#undef CMP_ONE + +StorePath SingleBuiltPath::outPath() const +{ + return std::visit( + overloaded{ + [](const SingleBuiltPath::Opaque & p) { return p.path; }, + [](const SingleBuiltPath::Built & b) { return b.output.second; }, + }, raw() + ); +} + +StorePathSet BuiltPath::outPaths() const +{ + return std::visit( + overloaded{ + [](const BuiltPath::Opaque & p) { return StorePathSet{p.path}; }, + [](const BuiltPath::Built & b) { + StorePathSet res; + for (auto & [_, path] : b.outputs) + res.insert(path); + return res; + }, + }, raw() + ); +} + +nlohmann::json BuiltPath::Built::toJSON(const Store & store) const +{ + nlohmann::json res; + res["drvPath"] = drvPath->toJSON(store); + for (const auto & [outputName, outputPath] : outputs) { + res["outputs"][outputName] = store.printStorePath(outputPath); + } + return res; +} + +nlohmann::json SingleBuiltPath::Built::toJSON(const Store & store) const +{ + nlohmann::json res; + res["drvPath"] = drvPath->toJSON(store); + auto & [outputName, outputPath] = output; + res["output"] = outputName; + res["outputPath"] = store.printStorePath(outputPath); + return res; +} + +nlohmann::json SingleBuiltPath::toJSON(const Store & store) const +{ + return std::visit([&](const auto & buildable) { + return buildable.toJSON(store); + }, raw()); +} + +nlohmann::json BuiltPath::toJSON(const Store & store) const +{ + return std::visit([&](const auto & buildable) { + return buildable.toJSON(store); + }, raw()); +} + +RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const +{ + RealisedPath::Set res; + std::visit( + overloaded{ + [&](const BuiltPath::Opaque & p) { res.insert(p.path); }, + [&](const BuiltPath::Built & p) { + auto drvHashes = + staticOutputHashes(store, store.readDerivation(p.drvPath->outPath())); + for (auto& [outputName, outputPath] : p.outputs) { + if (experimentalFeatureSettings.isEnabled( + Xp::CaDerivations)) { + auto drvOutput = get(drvHashes, outputName); + if (!drvOutput) + throw Error( + "the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)", + store.printStorePath(p.drvPath->outPath()), outputName); + auto thisRealisation = store.queryRealisation( + DrvOutput{*drvOutput, outputName}); + assert(thisRealisation); // We’ve built it, so we must + // have the realisation + res.insert(*thisRealisation); + } else { + res.insert(outputPath); + } + } + }, + }, + raw()); + return res; +} + +} diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh new file mode 100644 index 000000000..747bcc440 --- /dev/null +++ b/src/libcmd/built-path.hh @@ -0,0 +1,90 @@ +#include "derived-path.hh" +#include "realisation.hh" + +namespace nix { + +struct SingleBuiltPath; + +struct SingleBuiltPathBuilt { + ref drvPath; + std::pair output; + + std::string to_string(const Store & store) const; + static SingleBuiltPathBuilt parse(const Store & store, std::string_view, std::string_view); + nlohmann::json toJSON(const Store & store) const; + + DECLARE_CMP(SingleBuiltPathBuilt); +}; + +using _SingleBuiltPathRaw = std::variant< + DerivedPathOpaque, + SingleBuiltPathBuilt +>; + +struct SingleBuiltPath : _SingleBuiltPathRaw { + using Raw = _SingleBuiltPathRaw; + using Raw::Raw; + + using Opaque = DerivedPathOpaque; + using Built = SingleBuiltPathBuilt; + + inline const Raw & raw() const { + return static_cast(*this); + } + + StorePath outPath() const; + + static SingleBuiltPath parse(const Store & store, std::string_view); + nlohmann::json toJSON(const Store & store) const; +}; + +static inline ref staticDrv(StorePath drvPath) +{ + return make_ref(SingleBuiltPath::Opaque { drvPath }); +} + +/** + * A built derived path with hints in the form of optional concrete output paths. + * + * See 'BuiltPath' for more an explanation. + */ +struct BuiltPathBuilt { + ref drvPath; + std::map outputs; + + std::string to_string(const Store & store) const; + static BuiltPathBuilt parse(const Store & store, std::string_view, std::string_view); + nlohmann::json toJSON(const Store & store) const; + + DECLARE_CMP(BuiltPathBuilt); +}; + +using _BuiltPathRaw = std::variant< + DerivedPath::Opaque, + BuiltPathBuilt +>; + +/** + * A built path. Similar to a DerivedPath, but enriched with the corresponding + * output path(s). + */ +struct BuiltPath : _BuiltPathRaw { + using Raw = _BuiltPathRaw; + using Raw::Raw; + + using Opaque = DerivedPathOpaque; + using Built = BuiltPathBuilt; + + inline const Raw & raw() const { + return static_cast(*this); + } + + StorePathSet outPaths() const; + RealisedPath::Set toRealisedPaths(Store & store) const; + + nlohmann::json toJSON(const Store & store) const; +}; + +typedef std::vector BuiltPaths; + +} diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 7f97364a1..e36bda52f 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -1,3 +1,4 @@ +#include "eval-settings.hh" #include "common-eval-args.hh" #include "shared.hh" #include "filetransfer.hh" @@ -105,7 +106,9 @@ MixEvalArgs::MixEvalArgs() )", .category = category, .labels = {"path"}, - .handler = {[&](std::string s) { searchPath.push_back(s); }} + .handler = {[&](std::string s) { + searchPath.elements.emplace_back(SearchPath::Elem::parse(s)); + }} }); addFlag({ diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index b65cb5b20..6359b2579 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -3,6 +3,7 @@ #include "args.hh" #include "common-args.hh" +#include "search-path.hh" namespace nix { @@ -19,7 +20,7 @@ struct MixEvalArgs : virtual Args, virtual MixRepair Bindings * getAutoArgs(EvalState & state); - Strings searchPath; + SearchPath searchPath; std::optional evalStoreUrl; diff --git a/src/libcmd/installable-attr-path.cc b/src/libcmd/installable-attr-path.cc index b35ca2910..2f89eee02 100644 --- a/src/libcmd/installable-attr-path.cc +++ b/src/libcmd/installable-attr-path.cc @@ -92,7 +92,7 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths() for (auto & [drvPath, outputs] : byDrvPath) res.push_back({ .path = DerivedPath::Built { - .drvPath = drvPath, + .drvPath = makeConstantStorePathRef(drvPath), .outputs = outputs, }, .info = make_ref(ExtraPathInfoValue::Value { diff --git a/src/libcmd/installable-derived-path.cc b/src/libcmd/installable-derived-path.cc index 6ecf54b7c..b45641e8a 100644 --- a/src/libcmd/installable-derived-path.cc +++ b/src/libcmd/installable-derived-path.cc @@ -18,14 +18,7 @@ DerivedPathsWithInfo InstallableDerivedPath::toDerivedPaths() std::optional InstallableDerivedPath::getStorePath() { - return std::visit(overloaded { - [&](const DerivedPath::Built & bfd) { - return bfd.drvPath; - }, - [&](const DerivedPath::Opaque & bo) { - return bo.path; - }, - }, derivedPath.raw()); + return derivedPath.getBaseStorePath(); } InstallableDerivedPath InstallableDerivedPath::parse( @@ -42,7 +35,7 @@ InstallableDerivedPath InstallableDerivedPath::parse( // Remove this prior to stabilizing the new CLI. if (storePath.isDerivation()) { auto oldDerivedPath = DerivedPath::Built { - .drvPath = storePath, + .drvPath = makeConstantStorePathRef(storePath), .outputs = OutputsSpec::All { }, }; warn( @@ -55,8 +48,10 @@ InstallableDerivedPath InstallableDerivedPath::parse( }, // If the user did use ^, we just do exactly what is written. [&](const ExtendedOutputsSpec::Explicit & outputSpec) -> DerivedPath { + auto drv = make_ref(SingleDerivedPath::parse(*store, prefix)); + drvRequireExperiment(*drv); return DerivedPath::Built { - .drvPath = store->parseStorePath(prefix), + .drvPath = std::move(drv), .outputs = outputSpec, }; }, diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index 4da9b131b..1b169c3bd 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -118,7 +118,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() return {{ .path = DerivedPath::Built { - .drvPath = std::move(drvPath), + .drvPath = makeConstantStorePathRef(std::move(drvPath)), .outputs = std::visit(overloaded { [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec { std::set outputsToInstall; diff --git a/src/libcmd/installable-value.cc b/src/libcmd/installable-value.cc index 1eff293cc..08ad35105 100644 --- a/src/libcmd/installable-value.cc +++ b/src/libcmd/installable-value.cc @@ -55,7 +55,8 @@ std::optional InstallableValue::trySinglePathToDerivedPaths else if (v.type() == nString) { return {{ - .path = state->coerceToDerivedPath(pos, v, errorCtx), + .path = DerivedPath::fromSingle( + state->coerceToSingleDerivedPath(pos, v, errorCtx)), .info = make_ref(), }}; } diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index d9f3284f7..1611790f4 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -11,6 +11,7 @@ #include "derivations.hh" #include "eval-inline.hh" #include "eval.hh" +#include "eval-settings.hh" #include "get-drvs.hh" #include "store-api.hh" #include "shared.hh" @@ -672,6 +673,30 @@ ref SourceExprCommand::parseInstallable( return installables.front(); } +static SingleBuiltPath getBuiltPath(ref evalStore, ref store, const SingleDerivedPath & b) +{ + return std::visit( + overloaded{ + [&](const SingleDerivedPath::Opaque & bo) -> SingleBuiltPath { + return SingleBuiltPath::Opaque { bo.path }; + }, + [&](const SingleDerivedPath::Built & bfd) -> SingleBuiltPath { + auto drvPath = getBuiltPath(evalStore, store, *bfd.drvPath); + // Resolving this instead of `bfd` will yield the same result, but avoid duplicative work. + SingleDerivedPath::Built truncatedBfd { + .drvPath = makeConstantStorePathRef(drvPath.outPath()), + .output = bfd.output, + }; + auto outputPath = resolveDerivedPath(*store, truncatedBfd, &*evalStore); + return SingleBuiltPath::Built { + .drvPath = make_ref(std::move(drvPath)), + .output = { bfd.output, outputPath }, + }; + }, + }, + b.raw()); +} + std::vector Installable::build( ref evalStore, ref store, @@ -725,7 +750,10 @@ std::vector, BuiltPathWithResult>> Installable::build [&](const DerivedPath::Built & bfd) { auto outputs = resolveDerivedPath(*store, bfd, &*evalStore); res.push_back({aux.installable, { - .path = BuiltPath::Built { bfd.drvPath, outputs }, + .path = BuiltPath::Built { + .drvPath = make_ref(getBuiltPath(evalStore, store, *bfd.drvPath)), + .outputs = outputs, + }, .info = aux.info}}); }, [&](const DerivedPath::Opaque & bo) { @@ -754,7 +782,10 @@ std::vector, BuiltPathWithResult>> Installable::build for (auto & [outputName, realisation] : buildResult.builtOutputs) outputs.emplace(outputName, realisation.outPath); res.push_back({aux.installable, { - .path = BuiltPath::Built { bfd.drvPath, outputs }, + .path = BuiltPath::Built { + .drvPath = make_ref(getBuiltPath(evalStore, store, *bfd.drvPath)), + .outputs = outputs, + }, .info = aux.info, .result = buildResult}}); }, @@ -848,7 +879,7 @@ StorePathSet Installable::toDerivations( : throw Error("argument '%s' did not evaluate to a derivation", i->what())); }, [&](const DerivedPath::Built & bfd) { - drvPaths.insert(bfd.drvPath); + drvPaths.insert(resolveDerivedPath(*store, *bfd.drvPath)); }, }, b.path.raw()); diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index 42d6c7c7c..b0dc0dc02 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -5,6 +5,7 @@ #include "path.hh" #include "outputs-spec.hh" #include "derived-path.hh" +#include "built-path.hh" #include "store-api.hh" #include "build-result.hh" diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index d9c00ff91..3f1308f74 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -26,6 +26,7 @@ extern "C" { #include "eval.hh" #include "eval-cache.hh" #include "eval-inline.hh" +#include "eval-settings.hh" #include "attr-path.hh" #include "store-api.hh" #include "log-store.hh" @@ -68,7 +69,7 @@ struct NixRepl const Path historyFile; - NixRepl(const Strings & searchPath, nix::ref store,ref state, + NixRepl(const SearchPath & searchPath, nix::ref store,ref state, std::function getValues); virtual ~NixRepl(); @@ -104,7 +105,7 @@ std::string removeWhitespace(std::string s) } -NixRepl::NixRepl(const Strings & searchPath, nix::ref store, ref state, +NixRepl::NixRepl(const SearchPath & searchPath, nix::ref store, ref state, std::function getValues) : AbstractNixRepl(state) , debugTraceIndex(0) @@ -647,7 +648,7 @@ bool NixRepl::processLine(std::string line) if (command == ":b" || command == ":bl") { state->store->buildPaths({ DerivedPath::Built { - .drvPath = drvPath, + .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All { }, }, }); @@ -1024,7 +1025,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m std::unique_ptr AbstractNixRepl::create( - const Strings & searchPath, nix::ref store, ref state, + const SearchPath & searchPath, nix::ref store, ref state, std::function getValues) { return std::make_unique( @@ -1044,7 +1045,7 @@ void AbstractNixRepl::runSimple( NixRepl::AnnotatedValues values; return values; }; - const Strings & searchPath = {}; + SearchPath searchPath = {}; auto repl = std::make_unique( searchPath, openStore(), diff --git a/src/libcmd/repl.hh b/src/libcmd/repl.hh index 731c8e6db..6d88883fe 100644 --- a/src/libcmd/repl.hh +++ b/src/libcmd/repl.hh @@ -25,7 +25,7 @@ struct AbstractNixRepl typedef std::vector> AnnotatedValues; static std::unique_ptr create( - const Strings & searchPath, nix::ref store, ref state, + const SearchPath & searchPath, nix::ref store, ref state, std::function getValues); static void runSimple( diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 9e734e654..6a60ba87b 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -599,7 +599,7 @@ string_t AttrCursor::getStringWithContext() return d.drvPath; }, [&](const NixStringContextElem::Built & b) -> const StorePath & { - return b.drvPath; + return b.drvPath->getBaseStorePath(); }, [&](const NixStringContextElem::Opaque & o) -> const StorePath & { return o.path; diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc new file mode 100644 index 000000000..93b4a5289 --- /dev/null +++ b/src/libexpr/eval-settings.cc @@ -0,0 +1,102 @@ +#include "globals.hh" +#include "profiles.hh" +#include "eval.hh" +#include "eval-settings.hh" + +namespace nix { + +/* Very hacky way to parse $NIX_PATH, which is colon-separated, but + can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */ +static Strings parseNixPath(const std::string & s) +{ + Strings res; + + auto p = s.begin(); + + while (p != s.end()) { + auto start = p; + auto start2 = p; + + while (p != s.end() && *p != ':') { + if (*p == '=') start2 = p + 1; + ++p; + } + + if (p == s.end()) { + if (p != start) res.push_back(std::string(start, p)); + break; + } + + if (*p == ':') { + auto prefix = std::string(start2, s.end()); + if (EvalSettings::isPseudoUrl(prefix) || hasPrefix(prefix, "flake:")) { + ++p; + while (p != s.end() && *p != ':') ++p; + } + res.push_back(std::string(start, p)); + if (p == s.end()) break; + } + + ++p; + } + + return res; +} + +EvalSettings::EvalSettings() +{ + auto var = getEnv("NIX_PATH"); + if (var) nixPath = parseNixPath(*var); +} + +Strings EvalSettings::getDefaultNixPath() +{ + Strings res; + auto add = [&](const Path & p, const std::string & s = std::string()) { + if (pathAccessible(p)) { + if (s.empty()) { + res.push_back(p); + } else { + res.push_back(s + "=" + p); + } + } + }; + + if (!evalSettings.restrictEval && !evalSettings.pureEval) { + add(getNixDefExpr() + "/channels"); + add(rootChannelsDir() + "/nixpkgs", "nixpkgs"); + add(rootChannelsDir()); + } + + return res; +} + +bool EvalSettings::isPseudoUrl(std::string_view s) +{ + if (s.compare(0, 8, "channel:") == 0) return true; + size_t pos = s.find("://"); + if (pos == std::string::npos) return false; + std::string scheme(s, 0, pos); + return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh"; +} + +std::string EvalSettings::resolvePseudoUrl(std::string_view url) +{ + if (hasPrefix(url, "channel:")) + return "https://nixos.org/channels/" + std::string(url.substr(8)) + "/nixexprs.tar.xz"; + else + return std::string(url); +} + +EvalSettings evalSettings; + +static GlobalConfig::Register rEvalSettings(&evalSettings); + +Path getNixDefExpr() +{ + return settings.useXDGBaseDirectories + ? getStateDir() + "/nix/defexpr" + : getHome() + "/.nix-defexpr"; +} + +} diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh new file mode 100644 index 000000000..e6666061a --- /dev/null +++ b/src/libexpr/eval-settings.hh @@ -0,0 +1,103 @@ +#pragma once +#include "config.hh" + +namespace nix { + +struct EvalSettings : Config +{ + EvalSettings(); + + static Strings getDefaultNixPath(); + + static bool isPseudoUrl(std::string_view s); + + static std::string resolvePseudoUrl(std::string_view url); + + Setting enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", + "Whether builtin functions that allow executing native code should be enabled."}; + + Setting nixPath{ + this, getDefaultNixPath(), "nix-path", + R"( + List of directories to be searched for `<...>` file references + + In particular, outside of [pure evaluation mode](#conf-pure-evaluation), this determines the value of + [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath). + )"}; + + Setting restrictEval{ + this, false, "restrict-eval", + R"( + If set to `true`, the Nix evaluator will not allow access to any + files outside of the Nix search path (as set via the `NIX_PATH` + environment variable or the `-I` option), or to URIs outside of + [`allowed-uris`](../command-ref/conf-file.md#conf-allowed-uris). + The default is `false`. + )"}; + + Setting pureEval{this, false, "pure-eval", + R"( + Pure evaluation mode ensures that the result of Nix expressions is fully determined by explicitly declared inputs, and not influenced by external state: + + - Restrict file system and network access to files specified by cryptographic hash + - Disable [`bultins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) and [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime) + )" + }; + + Setting enableImportFromDerivation{ + this, true, "allow-import-from-derivation", + R"( + By default, Nix allows you to `import` from a derivation, allowing + building at evaluation time. With this option set to false, Nix will + throw an error when evaluating an expression that uses this feature, + allowing users to ensure their evaluation will not require any + builds to take place. + )"}; + + Setting allowedUris{this, {}, "allowed-uris", + R"( + A list of URI prefixes to which access is allowed in restricted + evaluation mode. For example, when set to + `https://github.com/NixOS`, builtin functions such as `fetchGit` are + allowed to access `https://github.com/NixOS/patchelf.git`. + )"}; + + Setting traceFunctionCalls{this, false, "trace-function-calls", + R"( + If set to `true`, the Nix evaluator will trace every function call. + Nix will print a log message at the "vomit" level for every function + entrance and function exit. + + function-trace entered undefined position at 1565795816999559622 + function-trace exited undefined position at 1565795816999581277 + function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150 + function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684 + + The `undefined position` means the function call is a builtin. + + Use the `contrib/stack-collapse.py` script distributed with the Nix + source code to convert the trace logs in to a format suitable for + `flamegraph.pl`. + )"}; + + Setting useEvalCache{this, true, "eval-cache", + "Whether to use the flake evaluation cache."}; + + Setting ignoreExceptionsDuringTry{this, false, "ignore-try", + R"( + If set to true, ignore exceptions inside 'tryEval' calls when evaluating nix expressions in + debug mode (using the --debugger flag). By default the debugger will pause on all exceptions. + )"}; + + Setting traceVerbose{this, false, "trace-verbose", + "Whether `builtins.traceVerbose` should trace its first argument when evaluated."}; +}; + +extern EvalSettings evalSettings; + +/** + * Conventionally part of the default nix path in impure mode. + */ +Path getNixDefExpr(); + +} diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 706a19024..3e521b732 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1,4 +1,5 @@ #include "eval.hh" +#include "eval-settings.hh" #include "hash.hh" #include "types.hh" #include "util.hh" @@ -420,44 +421,6 @@ void initGC() } -/* Very hacky way to parse $NIX_PATH, which is colon-separated, but - can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */ -static Strings parseNixPath(const std::string & s) -{ - Strings res; - - auto p = s.begin(); - - while (p != s.end()) { - auto start = p; - auto start2 = p; - - while (p != s.end() && *p != ':') { - if (*p == '=') start2 = p + 1; - ++p; - } - - if (p == s.end()) { - if (p != start) res.push_back(std::string(start, p)); - break; - } - - if (*p == ':') { - auto prefix = std::string(start2, s.end()); - if (EvalSettings::isPseudoUrl(prefix) || hasPrefix(prefix, "flake:")) { - ++p; - while (p != s.end() && *p != ':') ++p; - } - res.push_back(std::string(start, p)); - if (p == s.end()) break; - } - - ++p; - } - - return res; -} - ErrorBuilder & ErrorBuilder::atPos(PosIdx pos) { info.errPos = state.positions[pos]; @@ -498,7 +461,7 @@ ErrorBuilder & ErrorBuilder::withFrame(const Env & env, const Expr & expr) EvalState::EvalState( - const Strings & _searchPath, + const SearchPath & _searchPath, ref store, std::shared_ptr buildStore) : sWith(symbols.create("")) @@ -563,30 +526,32 @@ EvalState::EvalState( /* Initialise the Nix expression search path. */ if (!evalSettings.pureEval) { - for (auto & i : _searchPath) addToSearchPath(i); - for (auto & i : evalSettings.nixPath.get()) addToSearchPath(i); + for (auto & i : _searchPath.elements) + addToSearchPath(SearchPath::Elem {i}); + for (auto & i : evalSettings.nixPath.get()) + addToSearchPath(SearchPath::Elem::parse(i)); } if (evalSettings.restrictEval || evalSettings.pureEval) { allowedPaths = PathSet(); - for (auto & i : searchPath) { - auto r = resolveSearchPathElem(i); - if (!r.first) continue; + for (auto & i : searchPath.elements) { + auto r = resolveSearchPathPath(i.path); + if (!r) continue; - auto path = r.second; + auto path = *std::move(r); - if (store->isInStore(r.second)) { + if (store->isInStore(path)) { try { StorePathSet closure; - store->computeFSClosure(store->toStorePath(r.second).first, closure); + store->computeFSClosure(store->toStorePath(path).first, closure); for (auto & path : closure) allowPath(path); } catch (InvalidPath &) { - allowPath(r.second); + allowPath(path); } } else - allowPath(r.second); + allowPath(path); } } @@ -1066,17 +1031,18 @@ void EvalState::mkOutputString( Value & value, const StorePath & drvPath, const std::string outputName, - std::optional optOutputPath) + std::optional optOutputPath, + const ExperimentalFeatureSettings & xpSettings) { value.mkString( optOutputPath ? store->printStorePath(*std::move(optOutputPath)) /* Downstream we would substitute this for an actual path once we build the floating CA derivation */ - : DownstreamPlaceholder::unknownCaOutput(drvPath, outputName).render(), + : DownstreamPlaceholder::unknownCaOutput(drvPath, outputName, xpSettings).render(), NixStringContext { NixStringContextElem::Built { - .drvPath = drvPath, + .drvPath = makeConstantStorePathRef(drvPath), .output = outputName, } }); @@ -2333,7 +2299,7 @@ StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringCon } -std::pair EvalState::coerceToDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx) +std::pair EvalState::coerceToSingleDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx) { NixStringContext context; auto s = forceString(v, context, pos, errorCtx); @@ -2344,21 +2310,16 @@ std::pair EvalState::coerceToDerivedPathUnchecked s, csize) .withTrace(pos, errorCtx).debugThrow(); auto derivedPath = std::visit(overloaded { - [&](NixStringContextElem::Opaque && o) -> DerivedPath { - return DerivedPath::Opaque { - .path = std::move(o.path), - }; + [&](NixStringContextElem::Opaque && o) -> SingleDerivedPath { + return std::move(o); }, - [&](NixStringContextElem::DrvDeep &&) -> DerivedPath { + [&](NixStringContextElem::DrvDeep &&) -> SingleDerivedPath { error( "string '%s' has a context which refers to a complete source and binary closure. This is not supported at this time", s).withTrace(pos, errorCtx).debugThrow(); }, - [&](NixStringContextElem::Built && b) -> DerivedPath { - return DerivedPath::Built { - .drvPath = std::move(b.drvPath), - .outputs = OutputsSpec::Names { std::move(b.output) }, - }; + [&](NixStringContextElem::Built && b) -> SingleDerivedPath { + return std::move(b); }, }, ((NixStringContextElem &&) *context.begin()).raw()); return { @@ -2368,12 +2329,12 @@ std::pair EvalState::coerceToDerivedPathUnchecked } -DerivedPath EvalState::coerceToDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx) +SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx) { - auto [derivedPath, s_] = coerceToDerivedPathUnchecked(pos, v, errorCtx); + auto [derivedPath, s_] = coerceToSingleDerivedPathUnchecked(pos, v, errorCtx); auto s = s_; std::visit(overloaded { - [&](const DerivedPath::Opaque & o) { + [&](const SingleDerivedPath::Opaque & o) { auto sExpected = store->printStorePath(o.path); if (s != sExpected) error( @@ -2381,25 +2342,27 @@ DerivedPath EvalState::coerceToDerivedPath(const PosIdx pos, Value & v, std::str s, sExpected) .withTrace(pos, errorCtx).debugThrow(); }, - [&](const DerivedPath::Built & b) { - // TODO need derived path with single output to make this - // total. Will add as part of RFC 92 work and then this is - // cleaned up. - auto output = *std::get(b.outputs).begin(); - - auto drv = store->readDerivation(b.drvPath); - auto i = drv.outputs.find(output); - if (i == drv.outputs.end()) - throw Error("derivation '%s' does not have output '%s'", store->printStorePath(b.drvPath), output); - auto optOutputPath = i->second.path(*store, drv.name, output); - // This is testing for the case of CA derivations - auto sExpected = optOutputPath - ? store->printStorePath(*optOutputPath) - : DownstreamPlaceholder::unknownCaOutput(b.drvPath, output).render(); + [&](const SingleDerivedPath::Built & b) { + auto sExpected = std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) { + auto drv = store->readDerivation(o.path); + auto i = drv.outputs.find(b.output); + if (i == drv.outputs.end()) + throw Error("derivation '%s' does not have output '%s'", b.drvPath->to_string(*store), b.output); + auto optOutputPath = i->second.path(*store, drv.name, b.output); + // This is testing for the case of CA derivations + return optOutputPath + ? store->printStorePath(*optOutputPath) + : DownstreamPlaceholder::fromSingleDerivedPathBuilt(b).render(); + }, + [&](const SingleDerivedPath::Built & o) { + return DownstreamPlaceholder::fromSingleDerivedPathBuilt(b).render(); + }, + }, b.drvPath->raw()); if (s != sExpected) error( "string '%s' has context with the output '%s' from derivation '%s', but the string is not the right placeholder for this derivation output. It should be '%s'", - s, output, store->printStorePath(b.drvPath), sExpected) + s, b.output, b.drvPath->to_string(*store), sExpected) .withTrace(pos, errorCtx).debugThrow(); } }, derivedPath.raw()); @@ -2624,54 +2587,4 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) { } -EvalSettings::EvalSettings() -{ - auto var = getEnv("NIX_PATH"); - if (var) nixPath = parseNixPath(*var); -} - -Strings EvalSettings::getDefaultNixPath() -{ - Strings res; - auto add = [&](const Path & p, const std::string & s = std::string()) { - if (pathAccessible(p)) { - if (s.empty()) { - res.push_back(p); - } else { - res.push_back(s + "=" + p); - } - } - }; - - if (!evalSettings.restrictEval && !evalSettings.pureEval) { - add(settings.useXDGBaseDirectories ? getStateDir() + "/nix/defexpr/channels" : getHome() + "/.nix-defexpr/channels"); - add(rootChannelsDir() + "/nixpkgs", "nixpkgs"); - add(rootChannelsDir()); - } - - return res; -} - -bool EvalSettings::isPseudoUrl(std::string_view s) -{ - if (s.compare(0, 8, "channel:") == 0) return true; - size_t pos = s.find("://"); - if (pos == std::string::npos) return false; - std::string scheme(s, 0, pos); - return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh"; -} - -std::string EvalSettings::resolvePseudoUrl(std::string_view url) -{ - if (hasPrefix(url, "channel:")) - return "https://nixos.org/channels/" + std::string(url.substr(8)) + "/nixexprs.tar.xz"; - else - return std::string(url); -} - -EvalSettings evalSettings; - -static GlobalConfig::Register rEvalSettings(&evalSettings); - - } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index e3676c1b7..0268a2a12 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -9,6 +9,7 @@ #include "config.hh" #include "experimental-features.hh" #include "input-accessor.hh" +#include "search-path.hh" #include #include @@ -21,7 +22,7 @@ namespace nix { class Store; class EvalState; class StorePath; -struct DerivedPath; +struct SingleDerivedPath; enum RepairFlag : bool; @@ -122,15 +123,6 @@ std::string printValue(const EvalState & state, const Value & v); std::ostream & operator << (std::ostream & os, const ValueType t); -struct SearchPathElem -{ - std::string prefix; - // FIXME: maybe change this to an std::variant. - std::string path; -}; -typedef std::list SearchPath; - - /** * Initialise the Boehm GC, if applicable. */ @@ -317,7 +309,7 @@ private: SearchPath searchPath; - std::map> searchPathResolved; + std::map> searchPathResolved; /** * Cache used by checkSourcePath(). @@ -344,12 +336,12 @@ private: public: EvalState( - const Strings & _searchPath, + const SearchPath & _searchPath, ref store, std::shared_ptr buildStore = nullptr); ~EvalState(); - void addToSearchPath(const std::string & s); + void addToSearchPath(SearchPath::Elem && elem); SearchPath getSearchPath() { return searchPath; } @@ -431,12 +423,16 @@ public: * Look up a file in the search path. */ SourcePath findFile(const std::string_view path); - SourcePath findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos); + SourcePath findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos); /** + * Try to resolve a search path value (not the optinal key part) + * * If the specified search path element is a URI, download it. + * + * If it is not found, return `std::nullopt` */ - std::pair resolveSearchPathElem(const SearchPathElem & elem); + std::optional resolveSearchPathPath(const SearchPath::Path & path); /** * Evaluate an expression to normal form @@ -536,12 +532,12 @@ public: StorePath coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx); /** - * Part of `coerceToDerivedPath()` without any store IO which is exposed for unit testing only. + * Part of `coerceToSingleDerivedPath()` without any store IO which is exposed for unit testing only. */ - std::pair coerceToDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx); + std::pair coerceToSingleDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx); /** - * Coerce to `DerivedPath`. + * Coerce to `SingleDerivedPath`. * * Must be a string which is either a literal store path or a * "placeholder (see `DownstreamPlaceholder`). @@ -555,7 +551,7 @@ public: * source of truth, and ultimately tells us what we want, and then * we ensure the string corresponds to it. */ - DerivedPath coerceToDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx); + SingleDerivedPath coerceToSingleDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx); public: @@ -693,12 +689,15 @@ public: * be passed if and only if output store object is input-addressed. * Will be printed to form string if passed, otherwise a placeholder * will be used (see `DownstreamPlaceholder`). + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ void mkOutputString( Value & value, const StorePath & drvPath, const std::string outputName, - std::optional optOutputPath); + std::optional optOutputPath, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx); @@ -791,98 +790,6 @@ struct InvalidPathError : EvalError #endif }; -struct EvalSettings : Config -{ - EvalSettings(); - - static Strings getDefaultNixPath(); - - static bool isPseudoUrl(std::string_view s); - - static std::string resolvePseudoUrl(std::string_view url); - - Setting enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", - "Whether builtin functions that allow executing native code should be enabled."}; - - Setting nixPath{ - this, getDefaultNixPath(), "nix-path", - R"( - List of directories to be searched for `<...>` file references - - In particular, outside of [pure evaluation mode](#conf-pure-evaluation), this determines the value of - [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtin-constants-nixPath). - )"}; - - Setting restrictEval{ - this, false, "restrict-eval", - R"( - If set to `true`, the Nix evaluator will not allow access to any - files outside of the Nix search path (as set via the `NIX_PATH` - environment variable or the `-I` option), or to URIs outside of - [`allowed-uris`](../command-ref/conf-file.md#conf-allowed-uris). - The default is `false`. - )"}; - - Setting pureEval{this, false, "pure-eval", - R"( - Pure evaluation mode ensures that the result of Nix expressions is fully determined by explicitly declared inputs, and not influenced by external state: - - - Restrict file system and network access to files specified by cryptographic hash - - Disable [`bultins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) and [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime) - )" - }; - - Setting enableImportFromDerivation{ - this, true, "allow-import-from-derivation", - R"( - By default, Nix allows you to `import` from a derivation, allowing - building at evaluation time. With this option set to false, Nix will - throw an error when evaluating an expression that uses this feature, - allowing users to ensure their evaluation will not require any - builds to take place. - )"}; - - Setting allowedUris{this, {}, "allowed-uris", - R"( - A list of URI prefixes to which access is allowed in restricted - evaluation mode. For example, when set to - `https://github.com/NixOS`, builtin functions such as `fetchGit` are - allowed to access `https://github.com/NixOS/patchelf.git`. - )"}; - - Setting traceFunctionCalls{this, false, "trace-function-calls", - R"( - If set to `true`, the Nix evaluator will trace every function call. - Nix will print a log message at the "vomit" level for every function - entrance and function exit. - - function-trace entered undefined position at 1565795816999559622 - function-trace exited undefined position at 1565795816999581277 - function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150 - function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684 - - The `undefined position` means the function call is a builtin. - - Use the `contrib/stack-collapse.py` script distributed with the Nix - source code to convert the trace logs in to a format suitable for - `flamegraph.pl`. - )"}; - - Setting useEvalCache{this, true, "eval-cache", - "Whether to use the flake evaluation cache."}; - - Setting ignoreExceptionsDuringTry{this, false, "ignore-try", - R"( - If set to true, ignore exceptions inside 'tryEval' calls when evaluating nix expressions in - debug mode (using the --debugger flag). By default the debugger will pause on all exceptions. - )"}; - - Setting traceVerbose{this, false, "trace-verbose", - "Whether `builtins.traceVerbose` should trace its first argument when evaluated."}; -}; - -extern EvalSettings evalSettings; - static const std::string corepkgsPrefix{"/__corepkgs__/"}; template diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 8446b1f82..5042b24a9 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -1,5 +1,6 @@ #include "flake.hh" #include "eval.hh" +#include "eval-settings.hh" #include "lockfile.hh" #include "primops.hh" #include "eval-inline.hh" @@ -793,6 +794,101 @@ static RegisterPrimOp r2({ .experimentalFeature = Xp::Flakes, }); +static void prim_parseFlakeRef( + EvalState & state, + const PosIdx pos, + Value * * args, + Value & v) +{ + std::string flakeRefS(state.forceStringNoCtx(*args[0], pos, + "while evaluating the argument passed to builtins.parseFlakeRef")); + auto attrs = parseFlakeRef(flakeRefS, {}, true).toAttrs(); + auto binds = state.buildBindings(attrs.size()); + for (const auto & [key, value] : attrs) { + auto s = state.symbols.create(key); + auto & vv = binds.alloc(s); + std::visit(overloaded { + [&vv](const std::string & value) { vv.mkString(value); }, + [&vv](const uint64_t & value) { vv.mkInt(value); }, + [&vv](const Explicit & value) { vv.mkBool(value.t); } + }, value); + } + v.mkAttrs(binds); +} + +static RegisterPrimOp r3({ + .name = "__parseFlakeRef", + .args = {"flake-ref"}, + .doc = R"( + Parse a flake reference, and return its exploded form. + + For example: + ```nix + builtins.parseFlakeRef "github:NixOS/nixpkgs/23.05?dir=lib" + ``` + evaluates to: + ```nix + { dir = "lib"; owner = "NixOS"; ref = "23.05"; repo = "nixpkgs"; type = "github"; } + ``` + )", + .fun = prim_parseFlakeRef, + .experimentalFeature = Xp::Flakes, +}); + + +static void prim_flakeRefToString( + EvalState & state, + const PosIdx pos, + Value * * args, + Value & v) +{ + state.forceAttrs(*args[0], noPos, + "while evaluating the argument passed to builtins.flakeRefToString"); + fetchers::Attrs attrs; + for (const auto & attr : *args[0]->attrs) { + auto t = attr.value->type(); + if (t == nInt) { + attrs.emplace(state.symbols[attr.name], + (uint64_t) attr.value->integer); + } else if (t == nBool) { + attrs.emplace(state.symbols[attr.name], + Explicit { attr.value->boolean }); + } else if (t == nString) { + attrs.emplace(state.symbols[attr.name], + std::string(attr.value->str())); + } else { + state.error( + "flake reference attribute sets may only contain integers, Booleans, " + "and strings, but attribute '%s' is %s", + state.symbols[attr.name], + showType(*attr.value)).debugThrow(); + } + } + auto flakeRef = FlakeRef::fromAttrs(attrs); + v.mkString(flakeRef.to_string()); +} + +static RegisterPrimOp r4({ + .name = "__flakeRefToString", + .args = {"attrs"}, + .doc = R"( + Convert a flake reference from attribute set format to URL format. + + For example: + ```nix + builtins.flakeRefToString { + dir = "lib"; owner = "NixOS"; ref = "23.05"; repo = "nixpkgs"; type = "github"; + } + ``` + evaluates to + ```nix + "github:NixOS/nixpkgs/23.05?dir=lib" + ``` + )", + .fun = prim_flakeRefToString, + .experimentalFeature = Xp::Flakes, +}); + } Fingerprint LockedFlake::getFingerprint() const diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 08adbe0c9..d3fa1d557 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -105,7 +105,7 @@ std::pair parseFlakeRefWithFragment( }; return std::make_pair( - FlakeRef(Input::fromURL(parsedURL), ""), + FlakeRef(Input::fromURL(parsedURL, isFlake), ""), percentDecode(match.str(6))); } @@ -176,7 +176,7 @@ std::pair parseFlakeRefWithFragment( parsedURL.query.insert_or_assign("shallow", "1"); return std::make_pair( - FlakeRef(Input::fromURL(parsedURL), getOr(parsedURL.query, "dir", "")), + FlakeRef(Input::fromURL(parsedURL, isFlake), getOr(parsedURL.query, "dir", "")), fragment); } @@ -204,7 +204,7 @@ std::pair parseFlakeRefWithFragment( std::string fragment; std::swap(fragment, parsedURL.fragment); - auto input = Input::fromURL(parsedURL); + auto input = Input::fromURL(parsedURL, isFlake); input.parent = baseDir; return std::make_pair( diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 74bf270e7..f66f453e8 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -22,6 +22,7 @@ #include "nixexpr.hh" #include "eval.hh" +#include "eval-settings.hh" #include "globals.hh" namespace nix { @@ -137,6 +138,7 @@ static void addAttr(ExprAttrs * attrs, AttrPath && attrPath, dupAttr(state, ad.first, j2->second.pos, ad.second.pos); jAttrs->attrs.emplace(ad.first, ad.second); } + jAttrs->dynamicAttrs.insert(jAttrs->dynamicAttrs.end(), ae->dynamicAttrs.begin(), ae->dynamicAttrs.end()); } else { dupAttr(state, attrPath, pos, j->second.pos); } @@ -736,22 +738,9 @@ Expr * EvalState::parseStdin() } -void EvalState::addToSearchPath(const std::string & s) +void EvalState::addToSearchPath(SearchPath::Elem && elem) { - size_t pos = s.find('='); - std::string prefix; - Path path; - if (pos == std::string::npos) { - path = s; - } else { - prefix = std::string(s, 0, pos); - path = std::string(s, pos + 1); - } - - searchPath.emplace_back(SearchPathElem { - .prefix = prefix, - .path = path, - }); + searchPath.elements.emplace_back(std::move(elem)); } @@ -761,22 +750,19 @@ SourcePath EvalState::findFile(const std::string_view path) } -SourcePath EvalState::findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos) +SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos) { - for (auto & i : searchPath) { - std::string suffix; - if (i.prefix.empty()) - suffix = concatStrings("/", path); - else { - auto s = i.prefix.size(); - if (path.compare(0, s, i.prefix) != 0 || - (path.size() > s && path[s] != '/')) - continue; - suffix = path.size() == s ? "" : concatStrings("/", path.substr(s)); - } - auto r = resolveSearchPathElem(i); - if (!r.first) continue; - Path res = r.second + suffix; + for (auto & i : searchPath.elements) { + auto suffixOpt = i.prefix.suffixIfPotentialMatch(path); + + if (!suffixOpt) continue; + auto suffix = *suffixOpt; + + auto rOpt = resolveSearchPathPath(i.path); + if (!rOpt) continue; + auto r = *rOpt; + + Path res = suffix == "" ? r : concatStrings(r, "/", suffix); if (pathExists(res)) return CanonPath(canonPath(res)); } @@ -793,49 +779,53 @@ SourcePath EvalState::findFile(SearchPath & searchPath, const std::string_view p } -std::pair EvalState::resolveSearchPathElem(const SearchPathElem & elem) +std::optional EvalState::resolveSearchPathPath(const SearchPath::Path & value0) { - auto i = searchPathResolved.find(elem.path); + auto & value = value0.s; + auto i = searchPathResolved.find(value); if (i != searchPathResolved.end()) return i->second; - std::pair res; + std::optional res; - if (EvalSettings::isPseudoUrl(elem.path)) { + if (EvalSettings::isPseudoUrl(value)) { try { auto storePath = fetchers::downloadTarball( - store, EvalSettings::resolvePseudoUrl(elem.path), "source", false).tree.storePath; - res = { true, store->toRealPath(storePath) }; + store, EvalSettings::resolvePseudoUrl(value), "source", false).tree.storePath; + res = { store->toRealPath(storePath) }; } catch (FileTransferError & e) { logWarning({ - .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.path) + .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value) }); - res = { false, "" }; + res = std::nullopt; } } - else if (hasPrefix(elem.path, "flake:")) { + else if (hasPrefix(value, "flake:")) { experimentalFeatureSettings.require(Xp::Flakes); - auto flakeRef = parseFlakeRef(elem.path.substr(6), {}, true, false); - debug("fetching flake search path element '%s''", elem.path); + auto flakeRef = parseFlakeRef(value.substr(6), {}, true, false); + debug("fetching flake search path element '%s''", value); auto storePath = flakeRef.resolve(store).fetchTree(store).first.storePath; - res = { true, store->toRealPath(storePath) }; + res = { store->toRealPath(storePath) }; } else { - auto path = absPath(elem.path); + auto path = absPath(value); if (pathExists(path)) - res = { true, path }; + res = { path }; else { logWarning({ - .msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", elem.path) + .msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", value) }); - res = { false, "" }; + res = std::nullopt; } } - debug("resolved search path element '%s' to '%s'", elem.path, res.second); + if (res) + debug("resolved search path element '%s' to '%s'", value, *res); + else + debug("failed to resolve search path element '%s'", value); - searchPathResolved[elem.path] = res; + searchPathResolved[value] = res; return res; } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 123a06b2c..ab70047b9 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3,6 +3,7 @@ #include "downstream-placeholder.hh" #include "eval-inline.hh" #include "eval.hh" +#include "eval-settings.hh" #include "globals.hh" #include "json-to-value.hh" #include "names.hh" @@ -55,7 +56,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context) .drvPath = b.drvPath, .outputs = OutputsSpec::Names { b.output }, }); - ensureValid(b.drvPath); + ensureValid(b.drvPath->getBaseStorePath()); }, [&](const NixStringContextElem::Opaque & o) { auto ctxS = store->printStorePath(o.path); @@ -76,29 +77,32 @@ StringMap EvalState::realiseContext(const NixStringContext & context) if (!evalSettings.enableImportFromDerivation) debugThrowLastTrace(Error( "cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled", - store->printStorePath(drvs.begin()->drvPath))); + drvs.begin()->to_string(*store))); /* Build/substitute the context. */ std::vector buildReqs; for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d }); store->buildPaths(buildReqs); - /* Get all the output paths corresponding to the placeholders we had */ for (auto & drv : drvs) { auto outputs = resolveDerivedPath(*store, drv); for (auto & [outputName, outputPath] : outputs) { - res.insert_or_assign( - DownstreamPlaceholder::unknownCaOutput(drv.drvPath, outputName).render(), - store->printStorePath(outputPath) - ); - } - } - - /* Add the output of this derivations to the allowed - paths. */ - if (allowedPaths) { - for (auto & [_placeholder, outputPath] : res) { - allowPath(store->toRealPath(outputPath)); + /* Add the output of this derivations to the allowed + paths. */ + if (allowedPaths) { + allowPath(outputPath); + } + /* Get all the output paths corresponding to the placeholders we had */ + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { + res.insert_or_assign( + DownstreamPlaceholder::fromSingleDerivedPathBuilt( + SingleDerivedPath::Built { + .drvPath = drv.drvPath, + .output = outputName, + }).render(), + store->printStorePath(outputPath) + ); + } } } @@ -1251,7 +1255,10 @@ drvName, Bindings * attrs, Value & v) } }, [&](const NixStringContextElem::Built & b) { - drv.inputDrvs[b.drvPath].insert(b.output); + if (auto * p = std::get_if(&*b.drvPath)) + drv.inputDrvs[p->path].insert(b.output); + else + throw UnimplementedError("Dependencies on the outputs of dynamic derivations are not yet supported"); }, [&](const NixStringContextElem::Opaque & o) { drv.inputSrcs.insert(o.path); @@ -1300,9 +1307,10 @@ drvName, Bindings * attrs, Value & v) auto method = ingestionMethod.value_or(FileIngestionMethod::Flat); DerivationOutput::CAFixed dof { - .ca = ContentAddress::fromParts( - std::move(method), - std::move(h)), + .ca = ContentAddress { + .method = std::move(method), + .hash = std::move(h), + }, }; drv.env["out"] = state.store->printStorePath(dof.path(*state.store, drvName, "out")); @@ -1658,9 +1666,9 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V })); } - searchPath.emplace_back(SearchPathElem { - .prefix = prefix, - .path = path, + searchPath.elements.emplace_back(SearchPath::Elem { + .prefix = SearchPath::Prefix { .s = prefix }, + .path = SearchPath::Path { .s = path }, }); } @@ -2164,10 +2172,8 @@ static void addPath( std::optional expectedStorePath; if (expectedHash) expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo { - .hash = { - .method = method, - .hash = *expectedHash, - }, + .method = method, + .hash = *expectedHash, .references = {}, }); @@ -4329,12 +4335,12 @@ void EvalState::createBaseEnv() }); /* Add a value containing the current Nix expression search path. */ - mkList(v, searchPath.size()); + mkList(v, searchPath.elements.size()); int n = 0; - for (auto & i : searchPath) { + for (auto & i : searchPath.elements) { auto attrs = buildBindings(2); - attrs.alloc("path").mkString(i.path); - attrs.alloc("prefix").mkString(i.prefix); + attrs.alloc("path").mkString(i.path.s); + attrs.alloc("prefix").mkString(i.prefix.s); (v.listElems()[n++] = allocValue())->mkAttrs(attrs); } addConstant("__nixPath", v, { diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 8b3468009..bfc731744 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -106,7 +106,10 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args, contextInfos[std::move(d.drvPath)].allOutputs = true; }, [&](NixStringContextElem::Built && b) { - contextInfos[std::move(b.drvPath)].outputs.emplace_back(std::move(b.output)); + // FIXME should eventually show string context as is, no + // resolving here. + auto drvPath = resolveDerivedPath(*state.store, *b.drvPath); + contextInfos[std::move(drvPath)].outputs.emplace_back(std::move(b.output)); }, [&](NixStringContextElem::Opaque && o) { contextInfos[std::move(o.path)].path = true; @@ -222,7 +225,7 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar for (auto elem : iter->value->listItems()) { auto outputName = state.forceStringNoCtx(*elem, iter->pos, "while evaluating an output name within a string context"); context.emplace(NixStringContextElem::Built { - .drvPath = namePath, + .drvPath = makeConstantStorePathRef(namePath), .output = std::string { outputName }, }); } diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 322692b52..b9ff01c16 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -1,5 +1,6 @@ #include "primops.hh" #include "eval-inline.hh" +#include "eval-settings.hh" #include "store-api.hh" #include "fetchers.hh" #include "url.hh" diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 579a45f92..f040a3510 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -1,5 +1,6 @@ #include "primops.hh" #include "eval-inline.hh" +#include "eval-settings.hh" #include "store-api.hh" #include "fetchers.hh" #include "filetransfer.hh" @@ -254,10 +255,8 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v auto expectedPath = state.store->makeFixedOutputPath( name, FixedOutputInfo { - .hash = { - .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, - .hash = *expectedHash, - }, + .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, + .hash = *expectedHash, .references = {} }); diff --git a/src/libexpr/search-path.cc b/src/libexpr/search-path.cc new file mode 100644 index 000000000..36bb4c3a5 --- /dev/null +++ b/src/libexpr/search-path.cc @@ -0,0 +1,56 @@ +#include "search-path.hh" +#include "util.hh" + +namespace nix { + +std::optional SearchPath::Prefix::suffixIfPotentialMatch( + std::string_view path) const +{ + auto n = s.size(); + + /* Non-empty prefix and suffix must be separated by a /, or the + prefix is not a valid path prefix. */ + bool needSeparator = n > 0 && (path.size() - n) > 0; + + if (needSeparator && path[n] != '/') { + return std::nullopt; + } + + /* Prefix must be prefix of this path. */ + if (path.compare(0, n, s) != 0) { + return std::nullopt; + } + + /* Skip next path separator. */ + return { + path.substr(needSeparator ? n + 1 : n) + }; +} + + +SearchPath::Elem SearchPath::Elem::parse(std::string_view rawElem) +{ + size_t pos = rawElem.find('='); + + return SearchPath::Elem { + .prefix = Prefix { + .s = pos == std::string::npos + ? std::string { "" } + : std::string { rawElem.substr(0, pos) }, + }, + .path = Path { + .s = std::string { rawElem.substr(pos + 1) }, + }, + }; +} + + +SearchPath parseSearchPath(const Strings & rawElems) +{ + SearchPath res; + for (auto & rawElem : rawElems) + res.elements.emplace_back(SearchPath::Elem::parse(rawElem)); + return res; +} + +} diff --git a/src/libexpr/search-path.hh b/src/libexpr/search-path.hh new file mode 100644 index 000000000..ce78135b5 --- /dev/null +++ b/src/libexpr/search-path.hh @@ -0,0 +1,108 @@ +#pragma once +///@file + +#include + +#include "types.hh" +#include "comparator.hh" + +namespace nix { + +/** + * A "search path" is a list of ways look for something, used with + * `builtins.findFile` and `< >` lookup expressions. + */ +struct SearchPath +{ + /** + * A single element of a `SearchPath`. + * + * Each element is tried in succession when looking up a path. The first + * element to completely match wins. + */ + struct Elem; + + /** + * The first part of a `SearchPath::Elem` pair. + * + * Called a "prefix" because it takes the form of a prefix of a file + * path (first `n` path components). When looking up a path, to use + * a `SearchPath::Elem`, its `Prefix` must match the path. + */ + struct Prefix; + + /** + * The second part of a `SearchPath::Elem` pair. + * + * It is either a path or a URL (with certain restrictions / extra + * structure). + * + * If the prefix of the path we are looking up matches, we then + * check if the rest of the path points to something that exists + * within the directory denoted by this. If so, the + * `SearchPath::Elem` as a whole matches, and that *something* being + * pointed to by the rest of the path we are looking up is the + * result. + */ + struct Path; + + /** + * The list of search path elements. Each one is checked for a path + * when looking up. (The actual lookup entry point is in `EvalState` + * not in this class.) + */ + std::list elements; + + /** + * Parse a string into a `SearchPath` + */ + static SearchPath parse(const Strings & rawElems); +}; + +struct SearchPath::Prefix +{ + /** + * Underlying string + * + * @todo Should we normalize this when constructing a `SearchPath::Prefix`? + */ + std::string s; + + GENERATE_CMP(SearchPath::Prefix, me->s); + + /** + * If the path possibly matches this search path element, return the + * suffix that we should look for inside the resolved value of the + * element + * Note the double optionality in the name. While we might have a matching prefix, the suffix may not exist. + */ + std::optional suffixIfPotentialMatch(std::string_view path) const; +}; + +struct SearchPath::Path +{ + /** + * The location of a search path item, as a path or URL. + * + * @todo Maybe change this to `std::variant`. + */ + std::string s; + + GENERATE_CMP(SearchPath::Path, me->s); +}; + +struct SearchPath::Elem +{ + + Prefix prefix; + Path path; + + GENERATE_CMP(SearchPath::Elem, me->prefix, me->path); + + /** + * Parse a string into a `SearchPath::Elem` + */ + static SearchPath::Elem parse(std::string_view rawElem); +}; + +} diff --git a/src/libexpr/tests/derived-path.cc b/src/libexpr/tests/derived-path.cc index 8210efef2..2a5ca64f6 100644 --- a/src/libexpr/tests/derived-path.cc +++ b/src/libexpr/tests/derived-path.cc @@ -21,12 +21,12 @@ TEST_F(DerivedPathExpressionTest, force_init) RC_GTEST_FIXTURE_PROP( DerivedPathExpressionTest, prop_opaque_path_round_trip, - (const DerivedPath::Opaque & o)) + (const SingleDerivedPath::Opaque & o)) { auto * v = state.allocValue(); state.mkStorePathString(o.path, *v); - auto d = state.coerceToDerivedPath(noPos, *v, ""); - RC_ASSERT(DerivedPath { o } == d); + auto d = state.coerceToSingleDerivedPath(noPos, *v, ""); + RC_ASSERT(SingleDerivedPath { o } == d); } // TODO use DerivedPath::Built for parameter once it supports a single output @@ -37,14 +37,21 @@ RC_GTEST_FIXTURE_PROP( prop_built_path_placeholder_round_trip, (const StorePath & drvPath, const StorePathName & outputName)) { + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "ca-derivations"); + auto * v = state.allocValue(); - state.mkOutputString(*v, drvPath, outputName.name, std::nullopt); - auto [d, _] = state.coerceToDerivedPathUnchecked(noPos, *v, ""); - DerivedPath::Built b { - .drvPath = drvPath, - .outputs = OutputsSpec::Names { outputName.name }, + state.mkOutputString(*v, drvPath, outputName.name, std::nullopt, mockXpSettings); + auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, ""); + SingleDerivedPath::Built b { + .drvPath = makeConstantStorePathRef(drvPath), + .output = outputName.name, }; - RC_ASSERT(DerivedPath { b } == d); + RC_ASSERT(SingleDerivedPath { b } == d); } RC_GTEST_FIXTURE_PROP( @@ -54,12 +61,12 @@ RC_GTEST_FIXTURE_PROP( { auto * v = state.allocValue(); state.mkOutputString(*v, drvPath, outputName.name, outPath); - auto [d, _] = state.coerceToDerivedPathUnchecked(noPos, *v, ""); - DerivedPath::Built b { - .drvPath = drvPath, - .outputs = OutputsSpec::Names { outputName.name }, + auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, ""); + SingleDerivedPath::Built b { + .drvPath = makeConstantStorePathRef(drvPath), + .output = outputName.name, }; - RC_ASSERT(DerivedPath { b } == d); + RC_ASSERT(SingleDerivedPath { b } == d); } } /* namespace nix */ diff --git a/src/libexpr/tests/search-path.cc b/src/libexpr/tests/search-path.cc new file mode 100644 index 000000000..dbe7ab95f --- /dev/null +++ b/src/libexpr/tests/search-path.cc @@ -0,0 +1,90 @@ +#include +#include + +#include "search-path.hh" + +namespace nix { + +TEST(SearchPathElem, parse_justPath) { + ASSERT_EQ( + SearchPath::Elem::parse("foo"), + (SearchPath::Elem { + .prefix = SearchPath::Prefix { .s = "" }, + .path = SearchPath::Path { .s = "foo" }, + })); +} + +TEST(SearchPathElem, parse_emptyPrefix) { + ASSERT_EQ( + SearchPath::Elem::parse("=foo"), + (SearchPath::Elem { + .prefix = SearchPath::Prefix { .s = "" }, + .path = SearchPath::Path { .s = "foo" }, + })); +} + +TEST(SearchPathElem, parse_oneEq) { + ASSERT_EQ( + SearchPath::Elem::parse("foo=bar"), + (SearchPath::Elem { + .prefix = SearchPath::Prefix { .s = "foo" }, + .path = SearchPath::Path { .s = "bar" }, + })); +} + +TEST(SearchPathElem, parse_twoEqs) { + ASSERT_EQ( + SearchPath::Elem::parse("foo=bar=baz"), + (SearchPath::Elem { + .prefix = SearchPath::Prefix { .s = "foo" }, + .path = SearchPath::Path { .s = "bar=baz" }, + })); +} + + +TEST(SearchPathElem, suffixIfPotentialMatch_justPath) { + SearchPath::Prefix prefix { .s = "" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("any/thing"), std::optional { "any/thing" }); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_misleadingPrefix1) { + SearchPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("fooX"), std::nullopt); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_misleadingPrefix2) { + SearchPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("fooX/bar"), std::nullopt); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_partialPrefix) { + SearchPath::Prefix prefix { .s = "fooX" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo"), std::nullopt); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_exactPrefix) { + SearchPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo"), std::optional { "" }); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_multiKey) { + SearchPath::Prefix prefix { .s = "foo/bar" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo/bar/baz"), std::optional { "baz" }); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_trailingSlash) { + SearchPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo/"), std::optional { "" }); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_trailingDoubleSlash) { + SearchPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo//"), std::optional { "/" }); +} + +TEST(SearchPathElem, suffixIfPotentialMatch_trailingPath) { + SearchPath::Prefix prefix { .s = "foo" }; + ASSERT_EQ(prefix.suffixIfPotentialMatch("foo/bar/baz"), std::optional { "bar/baz" }); +} + +} diff --git a/src/libexpr/tests/value/context.cc b/src/libexpr/tests/value/context.cc index 0d9381577..c56b50b59 100644 --- a/src/libexpr/tests/value/context.cc +++ b/src/libexpr/tests/value/context.cc @@ -8,6 +8,8 @@ namespace nix { +// Test a few cases of invalid string context elements. + TEST(NixStringContextElemTest, empty_invalid) { EXPECT_THROW( NixStringContextElem::parse(""), @@ -38,6 +40,10 @@ TEST(NixStringContextElemTest, slash_invalid) { BadStorePath); } +/** + * Round trip (string <-> data structure) test for + * `NixStringContextElem::Opaque`. + */ TEST(NixStringContextElemTest, opaque) { std::string_view opaque = "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x"; auto elem = NixStringContextElem::parse(opaque); @@ -47,6 +53,10 @@ TEST(NixStringContextElemTest, opaque) { ASSERT_EQ(elem.to_string(), opaque); } +/** + * Round trip (string <-> data structure) test for + * `NixStringContextElem::DrvDeep`. + */ TEST(NixStringContextElemTest, drvDeep) { std::string_view drvDeep = "=g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; auto elem = NixStringContextElem::parse(drvDeep); @@ -56,28 +66,62 @@ TEST(NixStringContextElemTest, drvDeep) { ASSERT_EQ(elem.to_string(), drvDeep); } -TEST(NixStringContextElemTest, built) { +/** + * Round trip (string <-> data structure) test for a simpler + * `NixStringContextElem::Built`. + */ +TEST(NixStringContextElemTest, built_opaque) { std::string_view built = "!foo!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; auto elem = NixStringContextElem::parse(built); auto * p = std::get_if(&elem); ASSERT_TRUE(p); ASSERT_EQ(p->output, "foo"); - ASSERT_EQ(p->drvPath, StorePath { built.substr(5) }); + ASSERT_EQ(*p->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque { + .path = StorePath { built.substr(5) }, + })); ASSERT_EQ(elem.to_string(), built); } +/** + * Round trip (string <-> data structure) test for a more complex, + * inductive `NixStringContextElem::Built`. + */ +TEST(NixStringContextElemTest, built_built) { + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations"); + + std::string_view built = "!foo!bar!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; + auto elem = NixStringContextElem::parse(built, mockXpSettings); + auto * p = std::get_if(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->output, "foo"); + auto * drvPath = std::get_if(&*p->drvPath); + ASSERT_TRUE(drvPath); + ASSERT_EQ(drvPath->output, "bar"); + ASSERT_EQ(*drvPath->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque { + .path = StorePath { built.substr(9) }, + })); + ASSERT_EQ(elem.to_string(), built); +} + +/** + * Without the right experimental features enabled, we cannot parse a + * complex inductive string context element. + */ +TEST(NixStringContextElemTest, built_built_xp) { + ASSERT_THROW( + NixStringContextElem::parse("!foo!bar!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"), MissingExperimentalFeature); +} + } namespace rc { using namespace nix; -Gen Arbitrary::arbitrary() -{ - return gen::just(NixStringContextElem::Opaque { - .path = *gen::arbitrary(), - }); -} - Gen Arbitrary::arbitrary() { return gen::just(NixStringContextElem::DrvDeep { @@ -85,14 +129,6 @@ Gen Arbitrary::arb }); } -Gen Arbitrary::arbitrary() -{ - return gen::just(NixStringContextElem::Built { - .drvPath = *gen::arbitrary(), - .output = (*gen::arbitrary()).name, - }); -} - Gen Arbitrary::arbitrary() { switch (*gen::inRange(0, std::variant_size_v)) { diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 4996a5bde..ac3986c87 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -43,6 +43,7 @@ json printValueAsJSON(EvalState & state, bool strict, break; case nNull: + // already initialized as null break; case nAttrs: { @@ -59,7 +60,13 @@ json printValueAsJSON(EvalState & state, bool strict, names.emplace(state.symbols[j.name]); for (auto & j : names) { Attr & a(*v.attrs->find(state.symbols.create(j))); - out[j] = printValueAsJSON(state, strict, *a.value, a.pos, context, copyToStore); + try { + out[j] = printValueAsJSON(state, strict, *a.value, a.pos, context, copyToStore); + } catch (Error & e) { + e.addTrace(state.positions[a.pos], + hintfmt("while evaluating attribute '%1%'", j)); + throw; + } } } else return printValueAsJSON(state, strict, *i->value, i->pos, context, copyToStore); @@ -68,8 +75,17 @@ json printValueAsJSON(EvalState & state, bool strict, case nList: { out = json::array(); - for (auto elem : v.listItems()) - out.push_back(printValueAsJSON(state, strict, *elem, pos, context, copyToStore)); + int i = 0; + for (auto elem : v.listItems()) { + try { + out.push_back(printValueAsJSON(state, strict, *elem, pos, context, copyToStore)); + } catch (Error & e) { + e.addTrace({}, + hintfmt("while evaluating list element at index %1%", i)); + throw; + } + i++; + } break; } diff --git a/src/libexpr/value/context.cc b/src/libexpr/value/context.cc index f76fc76e4..d8116011e 100644 --- a/src/libexpr/value/context.cc +++ b/src/libexpr/value/context.cc @@ -4,29 +4,52 @@ namespace nix { -NixStringContextElem NixStringContextElem::parse(std::string_view s0) +NixStringContextElem NixStringContextElem::parse( + std::string_view s0, + const ExperimentalFeatureSettings & xpSettings) { std::string_view s = s0; + std::function parseRest; + parseRest = [&]() -> SingleDerivedPath { + // Case on whether there is a '!' + size_t index = s.find("!"); + if (index == std::string_view::npos) { + return SingleDerivedPath::Opaque { + .path = StorePath { s }, + }; + } else { + std::string output { s.substr(0, index) }; + // Advance string to parse after the '!' + s = s.substr(index + 1); + auto drv = make_ref(parseRest()); + drvRequireExperiment(*drv, xpSettings); + return SingleDerivedPath::Built { + .drvPath = std::move(drv), + .output = std::move(output), + }; + } + }; + if (s.size() == 0) { throw BadNixStringContextElem(s0, "String context element should never be an empty string"); } + switch (s.at(0)) { case '!': { - s = s.substr(1); // advance string to parse after first ! - size_t index = s.find("!"); - // This makes index + 1 safe. Index can be the length (one after index - // of last character), so given any valid character index --- a - // successful find --- we can add one. - if (index == std::string_view::npos) { + // Advance string to parse after the '!' + s = s.substr(1); + + // Find *second* '!' + if (s.find("!") == std::string_view::npos) { throw BadNixStringContextElem(s0, "String content element beginning with '!' should have a second '!'"); } - return NixStringContextElem::Built { - .drvPath = StorePath { s.substr(index + 1) }, - .output = std::string(s.substr(0, index)), - }; + + return std::visit( + [&](auto x) -> NixStringContextElem { return std::move(x); }, + parseRest()); } case '=': { return NixStringContextElem::DrvDeep { @@ -34,33 +57,51 @@ NixStringContextElem NixStringContextElem::parse(std::string_view s0) }; } default: { - return NixStringContextElem::Opaque { - .path = StorePath { s }, - }; + // Ensure no '!' + if (s.find("!") != std::string_view::npos) { + throw BadNixStringContextElem(s0, + "String content element not beginning with '!' should not have a second '!'"); + } + return std::visit( + [&](auto x) -> NixStringContextElem { return std::move(x); }, + parseRest()); } } } -std::string NixStringContextElem::to_string() const { - return std::visit(overloaded { +std::string NixStringContextElem::to_string() const +{ + std::string res; + + std::function toStringRest; + toStringRest = [&](auto & p) { + std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) { + res += o.path.to_string(); + }, + [&](const SingleDerivedPath::Built & o) { + res += o.output; + res += '!'; + toStringRest(*o.drvPath); + }, + }, p.raw()); + }; + + std::visit(overloaded { [&](const NixStringContextElem::Built & b) { - std::string res; res += '!'; - res += b.output; - res += '!'; - res += b.drvPath.to_string(); - return res; - }, - [&](const NixStringContextElem::DrvDeep & d) { - std::string res; - res += '='; - res += d.drvPath.to_string(); - return res; + toStringRest(b); }, [&](const NixStringContextElem::Opaque & o) { - return std::string { o.path.to_string() }; + toStringRest(o); + }, + [&](const NixStringContextElem::DrvDeep & d) { + res += '='; + res += d.drvPath.to_string(); }, }, raw()); + + return res; } } diff --git a/src/libexpr/value/context.hh b/src/libexpr/value/context.hh index 287ae08a9..a1b71695b 100644 --- a/src/libexpr/value/context.hh +++ b/src/libexpr/value/context.hh @@ -3,7 +3,7 @@ #include "util.hh" #include "comparator.hh" -#include "path.hh" +#include "derived-path.hh" #include @@ -31,11 +31,7 @@ public: * * Encoded as just the path: ‘’. */ -struct NixStringContextElem_Opaque { - StorePath path; - - GENERATE_CMP(NixStringContextElem_Opaque, me->path); -}; +typedef SingleDerivedPath::Opaque NixStringContextElem_Opaque; /** * Path to a derivation and its entire build closure. @@ -57,12 +53,7 @@ struct NixStringContextElem_DrvDeep { * * Encoded in the form ‘!!’. */ -struct NixStringContextElem_Built { - StorePath drvPath; - std::string output; - - GENERATE_CMP(NixStringContextElem_Built, me->drvPath, me->output); -}; +typedef SingleDerivedPath::Built NixStringContextElem_Built; using _NixStringContextElem_Raw = std::variant< NixStringContextElem_Opaque, @@ -93,8 +84,12 @@ struct NixStringContextElem : _NixStringContextElem_Raw { * - ‘’ * - ‘=’ * - ‘!!’ + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ - static NixStringContextElem parse(std::string_view s); + static NixStringContextElem parse( + std::string_view s, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); std::string to_string() const; }; diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 2860c1ceb..e683b9f80 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -13,9 +13,9 @@ void registerInputScheme(std::shared_ptr && inputScheme) inputSchemes->push_back(std::move(inputScheme)); } -Input Input::fromURL(const std::string & url) +Input Input::fromURL(const std::string & url, bool requireTree) { - return fromURL(parseURL(url)); + return fromURL(parseURL(url), requireTree); } static void fixupInput(Input & input) @@ -31,10 +31,10 @@ static void fixupInput(Input & input) input.locked = true; } -Input Input::fromURL(const ParsedURL & url) +Input Input::fromURL(const ParsedURL & url, bool requireTree) { for (auto & inputScheme : *inputSchemes) { - auto res = inputScheme->inputFromURL(url); + auto res = inputScheme->inputFromURL(url, requireTree); if (res) { res->scheme = inputScheme; fixupInput(*res); @@ -217,10 +217,8 @@ StorePath Input::computeStorePath(Store & store) const if (!narHash) throw Error("cannot compute store path for unlocked input '%s'", to_string()); return store.makeFixedOutputPath(getName(), FixedOutputInfo { - .hash = { - .method = FileIngestionMethod::Recursive, - .hash = *narHash, - }, + .method = FileIngestionMethod::Recursive, + .hash = *narHash, .references = {}, }); } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index d0738f619..6e10e9513 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -44,9 +44,9 @@ struct Input std::optional parent; public: - static Input fromURL(const std::string & url); + static Input fromURL(const std::string & url, bool requireTree = true); - static Input fromURL(const ParsedURL & url); + static Input fromURL(const ParsedURL & url, bool requireTree = true); static Input fromAttrs(Attrs && attrs); @@ -129,7 +129,7 @@ struct InputScheme virtual ~InputScheme() { } - virtual std::optional inputFromURL(const ParsedURL & url) const = 0; + virtual std::optional inputFromURL(const ParsedURL & url, bool requireTree) const = 0; virtual std::optional inputFromAttrs(const Attrs & attrs) const = 0; diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index e36a79975..17dcc18aa 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -256,7 +256,7 @@ std::pair fetchFromWorkdir(ref store, Input & input, co struct GitInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url) const override + std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override { if (url.scheme != "git" && url.scheme != "git+http" && diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 80598e7f8..291f457f0 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -30,7 +30,7 @@ struct GitArchiveInputScheme : InputScheme virtual std::optional> accessHeaderFromToken(const std::string & token) const = 0; - std::optional inputFromURL(const ParsedURL & url) const override + std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override { if (url.scheme != type()) return {}; diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index b99504a16..4874a43ff 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -7,7 +7,7 @@ std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); struct IndirectInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url) const override + std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override { if (url.scheme != "flake") return {}; diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 86e8f81f4..51fd1ed42 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -43,7 +43,7 @@ static std::string runHg(const Strings & args, const std::optional struct MercurialInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url) const override + std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override { if (url.scheme != "hg+http" && url.scheme != "hg+https" && diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 61541e69d..01f1be978 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -6,7 +6,7 @@ namespace nix::fetchers { struct PathInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url) const override + std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override { if (url.scheme != "path") return {}; diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index e42aca6db..107d38e92 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -77,10 +77,8 @@ DownloadFileResult downloadFile( *store, name, FixedOutputInfo { - .hash = { - .method = FileIngestionMethod::Flat, - .hash = hash, - }, + .method = FileIngestionMethod::Flat, + .hash = hash, .references = {}, }, hashString(htSHA256, sink.s), @@ -196,11 +194,11 @@ struct CurlInputScheme : InputScheme || hasSuffix(path, ".tar.zst"); } - virtual bool isValidURL(const ParsedURL & url) const = 0; + virtual bool isValidURL(const ParsedURL & url, bool requireTree) const = 0; - std::optional inputFromURL(const ParsedURL & _url) const override + std::optional inputFromURL(const ParsedURL & _url, bool requireTree) const override { - if (!isValidURL(_url)) + if (!isValidURL(_url, requireTree)) return std::nullopt; Input input; @@ -267,13 +265,13 @@ struct FileInputScheme : CurlInputScheme { const std::string inputType() const override { return "file"; } - bool isValidURL(const ParsedURL & url) const override + bool isValidURL(const ParsedURL & url, bool requireTree) const override { auto parsedUrlScheme = parseUrlScheme(url.scheme); return transportUrlSchemes.count(std::string(parsedUrlScheme.transport)) && (parsedUrlScheme.application - ? parsedUrlScheme.application.value() == inputType() - : !hasTarballExtension(url.path)); + ? parsedUrlScheme.application.value() == inputType() + : (!requireTree && !hasTarballExtension(url.path))); } std::pair fetch(ref store, const Input & input) override @@ -287,14 +285,14 @@ struct TarballInputScheme : CurlInputScheme { const std::string inputType() const override { return "tarball"; } - bool isValidURL(const ParsedURL & url) const override + bool isValidURL(const ParsedURL & url, bool requireTree) const override { auto parsedUrlScheme = parseUrlScheme(url.scheme); return transportUrlSchemes.count(std::string(parsedUrlScheme.transport)) && (parsedUrlScheme.application - ? parsedUrlScheme.application.value() == inputType() - : hasTarballExtension(url.path)); + ? parsedUrlScheme.application.value() == inputType() + : (requireTree || hasTarballExtension(url.path))); } std::pair fetch(ref store, const Input & _input) override diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index fcd763a9d..b4fea693f 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -309,10 +309,8 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n *this, name, FixedOutputInfo { - .hash = { - .method = method, - .hash = nar.first, - }, + .method = method, + .hash = nar.first, .references = { .others = references, // caller is not capable of creating a self-reference, because this is content-addressed without modulus @@ -428,10 +426,8 @@ StorePath BinaryCacheStore::addToStore( *this, name, FixedOutputInfo { - .hash = { - .method = method, - .hash = h, - }, + .method = method, + .hash = h, .references = { .others = references, // caller is not capable of creating a self-reference, because this is content-addressed without modulus @@ -465,8 +461,8 @@ StorePath BinaryCacheStore::addTextToStore( *this, std::string { name }, TextInfo { - { .hash = textHash }, - references, + .hash = textHash, + .references = references, }, nar.first, }; diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 5e37f7ecb..8bdf2f367 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -65,7 +65,7 @@ namespace nix { DerivationGoal::DerivationGoal(const StorePath & drvPath, const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs }) + : Goal(worker, DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = wantedOutputs }) , useDerivation(true) , drvPath(drvPath) , wantedOutputs(wantedOutputs) @@ -74,7 +74,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, state = &DerivationGoal::getDerivation; name = fmt( "building of '%s' from .drv file", - DerivedPath::Built { drvPath, wantedOutputs }.to_string(worker.store)); + DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store)); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); @@ -84,7 +84,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) - : Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs }) + : Goal(worker, DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = wantedOutputs }) , useDerivation(false) , drvPath(drvPath) , wantedOutputs(wantedOutputs) @@ -95,7 +95,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation state = &DerivationGoal::haveDerivation; name = fmt( "building of '%s' from in-memory derivation", - DerivedPath::Built { drvPath, drv.outputNames() }.to_string(worker.store)); + DerivedPath::Built { makeConstantStorePathRef(drvPath), drv.outputNames() }.to_string(worker.store)); trace("created"); mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); @@ -1490,7 +1490,7 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result) for (auto & outputName : outputs->second) { auto buildResult = dg->getBuildResult(DerivedPath::Built { - .drvPath = dg->drvPath, + .drvPath = makeConstantStorePathRef(dg->drvPath), .outputs = OutputsSpec::Names { outputName }, }); if (buildResult.success()) { diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 4aa4d6dca..e941b4e65 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -77,7 +77,7 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat try { worker.run(Goals{goal}); return goal->getBuildResult(DerivedPath::Built { - .drvPath = drvPath, + .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All {}, }); } catch (Error & e) { diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index ee66ee500..920097680 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1,5 +1,5 @@ #include "local-derivation-goal.hh" -#include "gc-store.hh" +#include "indirect-root-store.hh" #include "hook-instance.hh" #include "worker.hh" #include "builtins.hh" @@ -594,6 +594,10 @@ void LocalDerivationGoal::startBuilder() else dirsInChroot[i.substr(0, p)] = {i.substr(p + 1), optional}; } + if (hasPrefix(worker.store.storeDir, tmpDirInSandbox)) + { + throw Error("`sandbox-build-dir` must not contain the storeDir"); + } dirsInChroot[tmpDirInSandbox] = tmpDir; /* Add the closure of store paths to the chroot. */ @@ -908,15 +912,13 @@ void LocalDerivationGoal::startBuilder() openSlave(); /* Drop additional groups here because we can't do it - after we've created the new user namespace. FIXME: - this means that if we're not root in the parent - namespace, we can't drop additional groups; they will - be mapped to nogroup in the child namespace. There does - not seem to be a workaround for this. (But who can tell - from reading user_namespaces(7)?) - See also https://lwn.net/Articles/621612/. */ - if (getuid() == 0 && setgroups(0, 0) == -1) - throw SysError("setgroups failed"); + after we've created the new user namespace. */ + if (setgroups(0, 0) == -1) { + if (errno != EPERM) + throw SysError("setgroups failed"); + if (settings.requireDropSupplementaryGroups) + throw Error("setgroups failed. Set the require-drop-supplementary-groups option to false to skip this step."); + } ProcessOptions options; options.cloneFlags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; @@ -1170,6 +1172,19 @@ void LocalDerivationGoal::writeStructuredAttrs() } +static StorePath pathPartOfReq(const SingleDerivedPath & req) +{ + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) { + return bo.path; + }, + [&](const SingleDerivedPath::Built & bfd) { + return pathPartOfReq(*bfd.drvPath); + }, + }, req.raw()); +} + + static StorePath pathPartOfReq(const DerivedPath & req) { return std::visit(overloaded { @@ -1177,7 +1192,7 @@ static StorePath pathPartOfReq(const DerivedPath & req) return bo.path; }, [&](const DerivedPath::Built & bfd) { - return bfd.drvPath; + return pathPartOfReq(*bfd.drvPath); }, }, req.raw()); } @@ -1198,7 +1213,7 @@ struct RestrictedStoreConfig : virtual LocalFSStoreConfig /* A wrapper around LocalStore that only allows building/querying of paths that are in the input closures of the build or were added via recursive Nix calls. */ -struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore, public virtual GcStore +struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual IndirectRootStore, public virtual GcStore { ref next; @@ -1249,11 +1264,13 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo void queryReferrers(const StorePath & path, StorePathSet & referrers) override { } - std::map> queryPartialDerivationOutputMap(const StorePath & path) override + std::map> queryPartialDerivationOutputMap( + const StorePath & path, + Store * evalStore = nullptr) override { if (!goal.isAllowed(path)) throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path)); - return next->queryPartialDerivationOutputMap(path); + return next->queryPartialDerivationOutputMap(path, evalStore); } std::optional queryPathFromHashPart(const std::string & hashPart) override @@ -2303,7 +2320,6 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() bool discardReferences = false; if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { if (auto udr = get(*structuredAttrs, "unsafeDiscardReferences")) { - experimentalFeatureSettings.require(Xp::DiscardReferences); if (auto output = get(*udr, outputName)) { if (!output->is_boolean()) throw Error("attribute 'unsafeDiscardReferences.\"%s\"' of derivation '%s' must be a Boolean", outputName, drvPath.to_string()); @@ -2538,16 +2554,16 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() }, [&](const DerivationOutput::CAFixed & dof) { - auto wanted = dof.ca.getHash(); + auto & wanted = dof.ca.hash; auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating { - .method = dof.ca.getMethod(), + .method = dof.ca.method, .hashType = wanted.type, }); /* Check wanted hash */ assert(newInfo0.ca); - auto got = newInfo0.ca->getHash(); + auto & got = newInfo0.ca->hash; if (wanted != got) { /* Throw an error after registering the path as valid. */ diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index a9ca9cbbc..6779dbcf3 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -111,7 +111,10 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) { return std::visit(overloaded { [&](const DerivedPath::Built & bfd) -> GoalPtr { - return makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode); + if (auto bop = std::get_if(&*bfd.drvPath)) + return makeDerivationGoal(bop->path, bfd.outputs, buildMode); + else + throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented."); }, [&](const DerivedPath::Opaque & bo) -> GoalPtr { return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair); @@ -265,7 +268,7 @@ void Worker::run(const Goals & _topGoals) for (auto & i : _topGoals) { topGoals.insert(i); if (auto goal = dynamic_cast(i.get())) { - topPaths.push_back(DerivedPath::Built{goal->drvPath, goal->wantedOutputs}); + topPaths.push_back(DerivedPath::Built{makeConstantStorePathRef(goal->drvPath), goal->wantedOutputs}); } else if (auto goal = dynamic_cast(i.get())) { topPaths.push_back(DerivedPath::Opaque{goal->storePath}); } diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 04f7ac214..080456e18 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -4,11 +4,6 @@ namespace nix { -std::string FixedOutputHash::printMethodAlgo() const -{ - return makeFileIngestionPrefix(method) + printHashType(hash.type); -} - std::string makeFileIngestionPrefix(FileIngestionMethod m) { switch (m) { @@ -42,21 +37,6 @@ ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m) return method; } -std::string ContentAddress::render() const -{ - return std::visit(overloaded { - [](const TextHash & th) { - return "text:" - + th.hash.to_string(Base32, true); - }, - [](const FixedOutputHash & fsh) { - return "fixed:" - + makeFileIngestionPrefix(fsh.method) - + fsh.hash.to_string(Base32, true); - } - }, raw); -} - std::string ContentAddressMethod::render(HashType ht) const { return std::visit(overloaded { @@ -69,6 +49,20 @@ std::string ContentAddressMethod::render(HashType ht) const }, raw); } +std::string ContentAddress::render() const +{ + return std::visit(overloaded { + [](const TextIngestionMethod &) -> std::string { + return "text:"; + }, + [](const FileIngestionMethod & method) { + return "fixed:" + + makeFileIngestionPrefix(method); + }, + }, method.raw) + + this->hash.to_string(Base32, true); +} + /** * Parses content address strings up to the hash. */ @@ -118,22 +112,12 @@ ContentAddress ContentAddress::parse(std::string_view rawCa) { auto rest = rawCa; - auto [caMethod, hashType_] = parseContentAddressMethodPrefix(rest); - auto hashType = hashType_; // work around clang bug + auto [caMethod, hashType] = parseContentAddressMethodPrefix(rest); - return std::visit(overloaded { - [&](TextIngestionMethod &) { - return ContentAddress(TextHash { - .hash = Hash::parseNonSRIUnprefixed(rest, hashType) - }); - }, - [&](FileIngestionMethod & fim) { - return ContentAddress(FixedOutputHash { - .method = fim, - .hash = Hash::parseNonSRIUnprefixed(rest, hashType), - }); - }, - }, caMethod.raw); + return ContentAddress { + .method = std::move(caMethod).raw, + .hash = Hash::parseNonSRIUnprefixed(rest, hashType), + }; } std::pair ContentAddressMethod::parse(std::string_view caMethod) @@ -156,52 +140,10 @@ std::string renderContentAddress(std::optional ca) return ca ? ca->render() : ""; } -ContentAddress ContentAddress::fromParts( - ContentAddressMethod method, Hash hash) noexcept -{ - return std::visit(overloaded { - [&](TextIngestionMethod _) -> ContentAddress { - return TextHash { - .hash = std::move(hash), - }; - }, - [&](FileIngestionMethod m2) -> ContentAddress { - return FixedOutputHash { - .method = std::move(m2), - .hash = std::move(hash), - }; - }, - }, method.raw); -} - -ContentAddressMethod ContentAddress::getMethod() const -{ - return std::visit(overloaded { - [](const TextHash & th) -> ContentAddressMethod { - return TextIngestionMethod {}; - }, - [](const FixedOutputHash & fsh) -> ContentAddressMethod { - return fsh.method; - }, - }, raw); -} - -const Hash & ContentAddress::getHash() const -{ - return std::visit(overloaded { - [](const TextHash & th) -> auto & { - return th.hash; - }, - [](const FixedOutputHash & fsh) -> auto & { - return fsh.hash; - }, - }, raw); -} - std::string ContentAddress::printMethodAlgo() const { - return getMethod().renderPrefix() - + printHashType(getHash().type); + return method.renderPrefix() + + printHashType(hash.type); } bool StoreReferences::empty() const @@ -217,19 +159,20 @@ size_t StoreReferences::size() const ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const ContentAddress & ca) noexcept { return std::visit(overloaded { - [&](const TextHash & h) -> ContentAddressWithReferences { + [&](const TextIngestionMethod &) -> ContentAddressWithReferences { return TextInfo { - .hash = h, + .hash = ca.hash, .references = {}, }; }, - [&](const FixedOutputHash & h) -> ContentAddressWithReferences { + [&](const FileIngestionMethod & method) -> ContentAddressWithReferences { return FixedOutputInfo { - .hash = h, + .method = method, + .hash = ca.hash, .references = {}, }; }, - }, ca.raw); + }, ca.method.raw); } std::optional ContentAddressWithReferences::fromPartsOpt( @@ -241,7 +184,7 @@ std::optional ContentAddressWithReferences::fromPa return std::nullopt; return ContentAddressWithReferences { TextInfo { - .hash = { .hash = std::move(hash) }, + .hash = std::move(hash), .references = std::move(refs.others), } }; @@ -249,10 +192,8 @@ std::optional ContentAddressWithReferences::fromPa [&](FileIngestionMethod m2) -> std::optional { return ContentAddressWithReferences { FixedOutputInfo { - .hash = { - .method = m2, - .hash = std::move(hash), - }, + .method = m2, + .hash = std::move(hash), .references = std::move(refs), } }; @@ -267,7 +208,7 @@ ContentAddressMethod ContentAddressWithReferences::getMethod() const return TextIngestionMethod {}; }, [](const FixedOutputInfo & fsh) -> ContentAddressMethod { - return fsh.hash.method; + return fsh.method; }, }, raw); } @@ -276,10 +217,10 @@ Hash ContentAddressWithReferences::getHash() const { return std::visit(overloaded { [](const TextInfo & th) { - return th.hash.hash; + return th.hash; }, [](const FixedOutputInfo & fsh) { - return fsh.hash.hash; + return fsh.hash; }, }, raw); } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index e1e80448b..01b771e52 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -113,37 +113,6 @@ struct ContentAddressMethod * Mini content address */ -/** - * Somewhat obscure, used by \ref Derivation derivations and - * `builtins.toFile` currently. - */ -struct TextHash { - /** - * Hash of the contents of the text/file. - */ - Hash hash; - - GENERATE_CMP(TextHash, me->hash); -}; - -/** - * Used by most store objects that are content-addressed. - */ -struct FixedOutputHash { - /** - * How the file system objects are serialized - */ - FileIngestionMethod method; - /** - * Hash of that serialization - */ - Hash hash; - - std::string printMethodAlgo() const; - - GENERATE_CMP(FixedOutputHash, me->method, me->hash); -}; - /** * We've accumulated several types of content-addressed paths over the * years; fixed-output derivations support multiple hash algorithms and @@ -158,19 +127,17 @@ struct FixedOutputHash { */ struct ContentAddress { - typedef std::variant< - TextHash, - FixedOutputHash - > Raw; + /** + * How the file system objects are serialized + */ + ContentAddressMethod method; - Raw raw; + /** + * Hash of that serialization + */ + Hash hash; - GENERATE_CMP(ContentAddress, me->raw); - - /* The moral equivalent of `using Raw::Raw;` */ - ContentAddress(auto &&... arg) - : raw(std::forward(arg)...) - { } + GENERATE_CMP(ContentAddress, me->method, me->hash); /** * Compute the content-addressability assertion @@ -183,20 +150,6 @@ struct ContentAddress static std::optional parseOpt(std::string_view rawCaOpt); - /** - * Create a `ContentAddress` from 2 parts: - * - * @param method Way ingesting the file system data. - * - * @param hash Hash of ingested file system data. - */ - static ContentAddress fromParts( - ContentAddressMethod method, Hash hash) noexcept; - - ContentAddressMethod getMethod() const; - - const Hash & getHash() const; - std::string printMethodAlgo() const; }; @@ -219,7 +172,8 @@ std::string renderContentAddress(std::optional ca); * References to other store objects are tracked with store paths, self * references however are tracked with a boolean. */ -struct StoreReferences { +struct StoreReferences +{ /** * References to other store objects */ @@ -246,8 +200,13 @@ struct StoreReferences { }; // This matches the additional info that we need for makeTextPath -struct TextInfo { - TextHash hash; +struct TextInfo +{ + /** + * Hash of the contents of the text/file. + */ + Hash hash; + /** * References to other store objects only; self references * disallowed @@ -257,8 +216,18 @@ struct TextInfo { GENERATE_CMP(TextInfo, me->hash, me->references); }; -struct FixedOutputInfo { - FixedOutputHash hash; +struct FixedOutputInfo +{ + /** + * How the file system objects are serialized + */ + FileIngestionMethod method; + + /** + * Hash of that serialization + */ + Hash hash; + /** * References to other store objects or this one. */ diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index ad3dee1a2..8cbf6f044 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -7,6 +7,7 @@ #include "store-cast.hh" #include "gc-store.hh" #include "log-store.hh" +#include "indirect-root-store.hh" #include "path-with-outputs.hh" #include "finally.hh" #include "archive.hh" @@ -675,8 +676,8 @@ static void performOp(TunnelLogger * logger, ref store, Path path = absPath(readString(from)); logger->startWork(); - auto & gcStore = require(*store); - gcStore.addIndirectRoot(path); + auto & indirectRootStore = require(*store); + indirectRootStore.addIndirectRoot(path); logger->stopWork(); to << 1; diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 6f63685d4..f4e4980c2 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -232,9 +232,10 @@ static DerivationOutput parseDerivationOutput(const Store & store, validatePath(pathS); auto hash = Hash::parseNonSRIUnprefixed(hashS, hashType); return DerivationOutput::CAFixed { - .ca = ContentAddress::fromParts( - std::move(method), - std::move(hash)), + .ca = ContentAddress { + .method = std::move(method), + .hash = std::move(hash), + }, }; } else { experimentalFeatureSettings.require(Xp::CaDerivations); @@ -395,7 +396,7 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs, [&](const DerivationOutput::CAFixed & dof) { s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first))); s += ','; printUnquotedString(s, dof.ca.printMethodAlgo()); - s += ','; printUnquotedString(s, dof.ca.getHash().to_string(Base16, false)); + s += ','; printUnquotedString(s, dof.ca.hash.to_string(Base16, false)); }, [&](const DerivationOutput::CAFloating & dof) { s += ','; printUnquotedString(s, ""); @@ -628,7 +629,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut auto & dof = std::get(i.second.raw()); auto hash = hashString(htSHA256, "fixed:out:" + dof.ca.printMethodAlgo() + ":" - + dof.ca.getHash().to_string(Base16, false) + ":" + + dof.ca.hash.to_string(Base16, false) + ":" + store.printStorePath(dof.path(store, drv.name, i.first))); outputHashes.insert_or_assign(i.first, std::move(hash)); } @@ -780,7 +781,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr [&](const DerivationOutput::CAFixed & dof) { out << store.printStorePath(dof.path(store, drv.name, i.first)) << dof.ca.printMethodAlgo() - << dof.ca.getHash().to_string(Base16, false); + << dof.ca.hash.to_string(Base16, false); }, [&](const DerivationOutput::CAFloating & dof) { out << "" @@ -878,9 +879,11 @@ std::optional Derivation::tryResolve( for (auto & [inputDrv, inputOutputs] : inputDrvs) { for (auto & outputName : inputOutputs) { if (auto actualPath = get(inputDrvOutputs, { inputDrv, outputName })) { - inputRewrites.emplace( - DownstreamPlaceholder::unknownCaOutput(inputDrv, outputName).render(), - store.printStorePath(*actualPath)); + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { + inputRewrites.emplace( + DownstreamPlaceholder::unknownCaOutput(inputDrv, outputName).render(), + store.printStorePath(*actualPath)); + } resolved.inputSrcs.insert(*actualPath); } else { warn("output '%s' of input '%s' missing, aborting the resolving", @@ -970,7 +973,7 @@ nlohmann::json DerivationOutput::toJSON( [&](const DerivationOutput::CAFixed & dof) { res["path"] = store.printStorePath(dof.path(store, drvName, outputName)); res["hashAlgo"] = dof.ca.printMethodAlgo(); - res["hash"] = dof.ca.getHash().to_string(Base16, false); + res["hash"] = dof.ca.hash.to_string(Base16, false); // FIXME print refs? }, [&](const DerivationOutput::CAFloating & dof) { @@ -992,6 +995,7 @@ DerivationOutput DerivationOutput::fromJSON( const ExperimentalFeatureSettings & xpSettings) { std::set keys; + ensureType(_json, nlohmann::detail::value_t::object); auto json = (std::map) _json; for (const auto & [key, _] : json) @@ -1017,9 +1021,10 @@ DerivationOutput DerivationOutput::fromJSON( else if (keys == (std::set { "path", "hashAlgo", "hash" })) { auto [method, hashType] = methodAlgo(); auto dof = DerivationOutput::CAFixed { - .ca = ContentAddress::fromParts( - std::move(method), - Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType)), + .ca = ContentAddress { + .method = std::move(method), + .hash = Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType), + }, }; if (dof.path(store, drvName, outputName) != store.parseStorePath((std::string) json["path"])) throw Error("Path doesn't match derivation output"); @@ -1095,36 +1100,51 @@ Derivation Derivation::fromJSON( const Store & store, const nlohmann::json & json) { + using nlohmann::detail::value_t; + Derivation res; - res.name = json["name"]; + ensureType(json, value_t::object); - { - auto & outputsObj = json["outputs"]; + res.name = ensureType(valueAt(json, "name"), value_t::string); + + try { + auto & outputsObj = ensureType(valueAt(json, "outputs"), value_t::object); for (auto & [outputName, output] : outputsObj.items()) { res.outputs.insert_or_assign( outputName, DerivationOutput::fromJSON(store, res.name, outputName, output)); } + } catch (Error & e) { + e.addTrace({}, "while reading key 'outputs'"); + throw; } - { - auto & inputsList = json["inputSrcs"]; + try { + auto & inputsList = ensureType(valueAt(json, "inputSrcs"), value_t::array); for (auto & input : inputsList) res.inputSrcs.insert(store.parseStorePath(static_cast(input))); + } catch (Error & e) { + e.addTrace({}, "while reading key 'inputSrcs'"); + throw; } - { - auto & inputDrvsObj = json["inputDrvs"]; - for (auto & [inputDrvPath, inputOutputs] : inputDrvsObj.items()) + try { + auto & inputDrvsObj = ensureType(valueAt(json, "inputDrvs"), value_t::object); + for (auto & [inputDrvPath, inputOutputs] : inputDrvsObj.items()) { + ensureType(inputOutputs, value_t::array); res.inputDrvs[store.parseStorePath(inputDrvPath)] = static_cast(inputOutputs); + } + } catch (Error & e) { + e.addTrace({}, "while reading key 'inputDrvs'"); + throw; } - res.platform = json["system"]; - res.builder = json["builder"]; - res.args = json["args"]; - res.env = json["env"]; + res.platform = ensureType(valueAt(json, "system"), value_t::string); + res.builder = ensureType(valueAt(json, "builder"), value_t::string); + res.args = ensureType(valueAt(json, "args"), value_t::array); + res.env = ensureType(valueAt(json, "env"), value_t::object); return res; } diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 9a2ffda39..3594b7570 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -1,5 +1,4 @@ #include "derived-path.hh" -#include "derivations.hh" #include "store-api.hh" #include @@ -8,50 +7,83 @@ namespace nix { -nlohmann::json DerivedPath::Opaque::toJSON(ref store) const { +#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \ + bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \ + { \ + const MY_TYPE* me = this; \ + auto fields1 = std::make_tuple(*me->drvPath, me->FIELD); \ + me = &other; \ + auto fields2 = std::make_tuple(*me->drvPath, me->FIELD); \ + return fields1 COMPARATOR fields2; \ + } +#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \ + CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <) + +#define FIELD_TYPE std::string +CMP(SingleDerivedPath, SingleDerivedPathBuilt, output) +#undef FIELD_TYPE + +#define FIELD_TYPE OutputsSpec +CMP(SingleDerivedPath, DerivedPathBuilt, outputs) +#undef FIELD_TYPE + +#undef CMP +#undef CMP_ONE + +nlohmann::json DerivedPath::Opaque::toJSON(const Store & store) const +{ + return store.printStorePath(path); +} + +nlohmann::json SingleDerivedPath::Built::toJSON(Store & store) const { nlohmann::json res; - res["path"] = store->printStorePath(path); + res["drvPath"] = drvPath->toJSON(store); + // Fallback for the input-addressed derivation case: We expect to always be + // able to print the output paths, so let’s do it + // FIXME try-resolve on drvPath + const auto outputMap = store.queryPartialDerivationOutputMap(resolveDerivedPath(store, *drvPath)); + res["output"] = output; + auto outputPathIter = outputMap.find(output); + if (outputPathIter == outputMap.end()) + res["outputPath"] = nullptr; + else if (std::optional p = outputPathIter->second) + res["outputPath"] = store.printStorePath(*p); + else + res["outputPath"] = nullptr; return res; } -nlohmann::json DerivedPath::Built::toJSON(ref store) const { +nlohmann::json DerivedPath::Built::toJSON(Store & store) const { nlohmann::json res; - res["drvPath"] = store->printStorePath(drvPath); + res["drvPath"] = drvPath->toJSON(store); // Fallback for the input-addressed derivation case: We expect to always be // able to print the output paths, so let’s do it - const auto outputMap = store->queryPartialDerivationOutputMap(drvPath); + // FIXME try-resolve on drvPath + const auto outputMap = store.queryPartialDerivationOutputMap(resolveDerivedPath(store, *drvPath)); for (const auto & [output, outputPathOpt] : outputMap) { if (!outputs.contains(output)) continue; if (outputPathOpt) - res["outputs"][output] = store->printStorePath(*outputPathOpt); + res["outputs"][output] = store.printStorePath(*outputPathOpt); else res["outputs"][output] = nullptr; } return res; } -nlohmann::json BuiltPath::Built::toJSON(ref store) const { - nlohmann::json res; - res["drvPath"] = store->printStorePath(drvPath); - for (const auto& [output, path] : outputs) { - res["outputs"][output] = store->printStorePath(path); - } - return res; +nlohmann::json SingleDerivedPath::toJSON(Store & store) const +{ + return std::visit([&](const auto & buildable) { + return buildable.toJSON(store); + }, raw()); } -StorePathSet BuiltPath::outPaths() const +nlohmann::json DerivedPath::toJSON(Store & store) const { - return std::visit( - overloaded{ - [](const BuiltPath::Opaque & p) { return StorePathSet{p.path}; }, - [](const BuiltPath::Built & b) { - StorePathSet res; - for (auto & [_, path] : b.outputs) - res.insert(path); - return res; - }, - }, raw() - ); + return std::visit([&](const auto & buildable) { + return buildable.toJSON(store); + }, raw()); } std::string DerivedPath::Opaque::to_string(const Store & store) const @@ -59,25 +91,49 @@ std::string DerivedPath::Opaque::to_string(const Store & store) const return store.printStorePath(path); } +std::string SingleDerivedPath::Built::to_string(const Store & store) const +{ + return drvPath->to_string(store) + "^" + output; +} + +std::string SingleDerivedPath::Built::to_string_legacy(const Store & store) const +{ + return drvPath->to_string(store) + "!" + output; +} + std::string DerivedPath::Built::to_string(const Store & store) const { - return store.printStorePath(drvPath) + return drvPath->to_string(store) + '^' + outputs.to_string(); } std::string DerivedPath::Built::to_string_legacy(const Store & store) const { - return store.printStorePath(drvPath) - + '!' + return drvPath->to_string_legacy(store) + + "!" + outputs.to_string(); } +std::string SingleDerivedPath::to_string(const Store & store) const +{ + return std::visit( + [&](const auto & req) { return req.to_string(store); }, + raw()); +} + std::string DerivedPath::to_string(const Store & store) const +{ + return std::visit( + [&](const auto & req) { return req.to_string(store); }, + raw()); +} + +std::string SingleDerivedPath::to_string_legacy(const Store & store) const { return std::visit(overloaded { - [&](const DerivedPath::Built & req) { return req.to_string(store); }, - [&](const DerivedPath::Opaque & req) { return req.to_string(store); }, + [&](const SingleDerivedPath::Built & req) { return req.to_string_legacy(store); }, + [&](const SingleDerivedPath::Opaque & req) { return req.to_string(store); }, }, this->raw()); } @@ -95,61 +151,156 @@ DerivedPath::Opaque DerivedPath::Opaque::parse(const Store & store, std::string_ return {store.parseStorePath(s)}; } -DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_view drvS, std::string_view outputsS) +void drvRequireExperiment( + const SingleDerivedPath & drv, + const ExperimentalFeatureSettings & xpSettings) { + std::visit(overloaded { + [&](const SingleDerivedPath::Opaque &) { + // plain drv path; no experimental features required. + }, + [&](const SingleDerivedPath::Built &) { + xpSettings.require(Xp::DynamicDerivations); + }, + }, drv.raw()); +} + +SingleDerivedPath::Built SingleDerivedPath::Built::parse( + const Store & store, ref drv, + std::string_view output, + const ExperimentalFeatureSettings & xpSettings) +{ + drvRequireExperiment(*drv, xpSettings); return { - .drvPath = store.parseStorePath(drvS), + .drvPath = drv, + .output = std::string { output }, + }; +} + +DerivedPath::Built DerivedPath::Built::parse( + const Store & store, ref drv, + std::string_view outputsS, + const ExperimentalFeatureSettings & xpSettings) +{ + drvRequireExperiment(*drv, xpSettings); + return { + .drvPath = drv, .outputs = OutputsSpec::parse(outputsS), }; } -static inline DerivedPath parseWith(const Store & store, std::string_view s, std::string_view separator) +static SingleDerivedPath parseWithSingle( + const Store & store, std::string_view s, std::string_view separator, + const ExperimentalFeatureSettings & xpSettings) { - size_t n = s.find(separator); + size_t n = s.rfind(separator); + return n == s.npos + ? (SingleDerivedPath) SingleDerivedPath::Opaque::parse(store, s) + : (SingleDerivedPath) SingleDerivedPath::Built::parse(store, + make_ref(parseWithSingle( + store, + s.substr(0, n), + separator, + xpSettings)), + s.substr(n + 1), + xpSettings); +} + +SingleDerivedPath SingleDerivedPath::parse( + const Store & store, + std::string_view s, + const ExperimentalFeatureSettings & xpSettings) +{ + return parseWithSingle(store, s, "^", xpSettings); +} + +SingleDerivedPath SingleDerivedPath::parseLegacy( + const Store & store, + std::string_view s, + const ExperimentalFeatureSettings & xpSettings) +{ + return parseWithSingle(store, s, "!", xpSettings); +} + +static DerivedPath parseWith( + const Store & store, std::string_view s, std::string_view separator, + const ExperimentalFeatureSettings & xpSettings) +{ + size_t n = s.rfind(separator); return n == s.npos ? (DerivedPath) DerivedPath::Opaque::parse(store, s) - : (DerivedPath) DerivedPath::Built::parse(store, s.substr(0, n), s.substr(n + 1)); + : (DerivedPath) DerivedPath::Built::parse(store, + make_ref(parseWithSingle( + store, + s.substr(0, n), + separator, + xpSettings)), + s.substr(n + 1), + xpSettings); } -DerivedPath DerivedPath::parse(const Store & store, std::string_view s) +DerivedPath DerivedPath::parse( + const Store & store, + std::string_view s, + const ExperimentalFeatureSettings & xpSettings) { - return parseWith(store, s, "^"); + return parseWith(store, s, "^", xpSettings); } -DerivedPath DerivedPath::parseLegacy(const Store & store, std::string_view s) +DerivedPath DerivedPath::parseLegacy( + const Store & store, + std::string_view s, + const ExperimentalFeatureSettings & xpSettings) { - return parseWith(store, s, "!"); + return parseWith(store, s, "!", xpSettings); } -RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const +DerivedPath DerivedPath::fromSingle(const SingleDerivedPath & req) { - RealisedPath::Set res; - std::visit( - overloaded{ - [&](const BuiltPath::Opaque & p) { res.insert(p.path); }, - [&](const BuiltPath::Built & p) { - auto drvHashes = - staticOutputHashes(store, store.readDerivation(p.drvPath)); - for (auto& [outputName, outputPath] : p.outputs) { - if (experimentalFeatureSettings.isEnabled( - Xp::CaDerivations)) { - auto drvOutput = get(drvHashes, outputName); - if (!drvOutput) - throw Error( - "the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)", - store.printStorePath(p.drvPath), outputName); - auto thisRealisation = store.queryRealisation( - DrvOutput{*drvOutput, outputName}); - assert(thisRealisation); // We’ve built it, so we must - // have the realisation - res.insert(*thisRealisation); - } else { - res.insert(outputPath); - } - } - }, + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) -> DerivedPath { + return o; }, - raw()); - return res; + [&](const SingleDerivedPath::Built & b) -> DerivedPath { + return DerivedPath::Built { + .drvPath = b.drvPath, + .outputs = OutputsSpec::Names { b.output }, + }; + }, + }, req.raw()); } + +const StorePath & SingleDerivedPath::Built::getBaseStorePath() const +{ + return drvPath->getBaseStorePath(); +} + +const StorePath & DerivedPath::Built::getBaseStorePath() const +{ + return drvPath->getBaseStorePath(); +} + +template +static inline const StorePath & getBaseStorePath_(const DP & derivedPath) +{ + return std::visit(overloaded { + [&](const typename DP::Built & bfd) -> auto & { + return bfd.drvPath->getBaseStorePath(); + }, + [&](const typename DP::Opaque & bo) -> auto & { + return bo.path; + }, + }, derivedPath.raw()); +} + +const StorePath & SingleDerivedPath::getBaseStorePath() const +{ + return getBaseStorePath_(*this); +} + +const StorePath & DerivedPath::getBaseStorePath() const +{ + return getBaseStorePath_(*this); +} + } diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 5f7acbebc..ec30dac61 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -3,7 +3,6 @@ #include "util.hh" #include "path.hh" -#include "realisation.hh" #include "outputs-spec.hh" #include "comparator.hh" @@ -25,28 +24,37 @@ class Store; struct DerivedPathOpaque { StorePath path; - nlohmann::json toJSON(ref store) const; std::string to_string(const Store & store) const; static DerivedPathOpaque parse(const Store & store, std::string_view); + nlohmann::json toJSON(const Store & store) const; GENERATE_CMP(DerivedPathOpaque, me->path); }; +struct SingleDerivedPath; + /** - * A derived path that is built from a derivation + * A single derived path that is built from a derivation * - * Built derived paths are pair of a derivation and some output names. - * They are evaluated by building the derivation, and then replacing the - * output names with the resulting outputs. - * - * Note that does mean a derived store paths evaluates to multiple - * opaque paths, which is sort of icky as expressions are supposed to - * evaluate to single values. Perhaps this should have just a single - * output name. + * Built derived paths are pair of a derivation and an output name. They are + * evaluated by building the derivation, and then taking the resulting output + * path of the given output name. */ -struct DerivedPathBuilt { - StorePath drvPath; - OutputsSpec outputs; +struct SingleDerivedPathBuilt { + ref drvPath; + std::string output; + + /** + * Get the store path this is ultimately derived from (by realising + * and projecting outputs). + * + * Note that this is *not* a property of the store object being + * referred to, but just of this path --- how we happened to be + * referring to that store object. In other words, this means this + * function breaks "referential transparency". It should therefore + * be used only with great care. + */ + const StorePath & getBaseStorePath() const; /** * Uses `^` as the separator @@ -58,11 +66,139 @@ struct DerivedPathBuilt { std::string to_string_legacy(const Store & store) const; /** * The caller splits on the separator, so it works for both variants. + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ - static DerivedPathBuilt parse(const Store & store, std::string_view drvPath, std::string_view outputs); - nlohmann::json toJSON(ref store) const; + static SingleDerivedPathBuilt parse( + const Store & store, ref drvPath, + std::string_view outputs, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + nlohmann::json toJSON(Store & store) const; - GENERATE_CMP(DerivedPathBuilt, me->drvPath, me->outputs); + DECLARE_CMP(SingleDerivedPathBuilt); +}; + +using _SingleDerivedPathRaw = std::variant< + DerivedPathOpaque, + SingleDerivedPathBuilt +>; + +/** + * A "derived path" is a very simple sort of expression (not a Nix + * language expression! But an expression in a the general sense) that + * evaluates to (concrete) store path. It is either: + * + * - opaque, in which case it is just a concrete store path with + * possibly no known derivation + * + * - built, in which case it is a pair of a derivation path and an + * output name. + */ +struct SingleDerivedPath : _SingleDerivedPathRaw { + using Raw = _SingleDerivedPathRaw; + using Raw::Raw; + + using Opaque = DerivedPathOpaque; + using Built = SingleDerivedPathBuilt; + + inline const Raw & raw() const { + return static_cast(*this); + } + + /** + * Get the store path this is ultimately derived from (by realising + * and projecting outputs). + * + * Note that this is *not* a property of the store object being + * referred to, but just of this path --- how we happened to be + * referring to that store object. In other words, this means this + * function breaks "referential transparency". It should therefore + * be used only with great care. + */ + const StorePath & getBaseStorePath() const; + + /** + * Uses `^` as the separator + */ + std::string to_string(const Store & store) const; + /** + * Uses `!` as the separator + */ + std::string to_string_legacy(const Store & store) const; + /** + * Uses `^` as the separator + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ + static SingleDerivedPath parse( + const Store & store, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + /** + * Uses `!` as the separator + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ + static SingleDerivedPath parseLegacy( + const Store & store, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + nlohmann::json toJSON(Store & store) const; +}; + +static inline ref makeConstantStorePathRef(StorePath drvPath) +{ + return make_ref(SingleDerivedPath::Opaque { drvPath }); +} + +/** + * A set of derived paths that are built from a derivation + * + * Built derived paths are pair of a derivation and some output names. + * They are evaluated by building the derivation, and then replacing the + * output names with the resulting outputs. + * + * Note that does mean a derived store paths evaluates to multiple + * opaque paths, which is sort of icky as expressions are supposed to + * evaluate to single values. Perhaps this should have just a single + * output name. + */ +struct DerivedPathBuilt { + ref drvPath; + OutputsSpec outputs; + + /** + * Get the store path this is ultimately derived from (by realising + * and projecting outputs). + * + * Note that this is *not* a property of the store object being + * referred to, but just of this path --- how we happened to be + * referring to that store object. In other words, this means this + * function breaks "referential transparency". It should therefore + * be used only with great care. + */ + const StorePath & getBaseStorePath() const; + + /** + * Uses `^` as the separator + */ + std::string to_string(const Store & store) const; + /** + * Uses `!` as the separator + */ + std::string to_string_legacy(const Store & store) const; + /** + * The caller splits on the separator, so it works for both variants. + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ + static DerivedPathBuilt parse( + const Store & store, ref, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + nlohmann::json toJSON(Store & store) const; + + DECLARE_CMP(DerivedPathBuilt); }; using _DerivedPathRaw = std::variant< @@ -72,13 +208,13 @@ using _DerivedPathRaw = std::variant< /** * A "derived path" is a very simple sort of expression that evaluates - * to (concrete) store path. It is either: + * to one or more (concrete) store paths. It is either: * - * - opaque, in which case it is just a concrete store path with + * - opaque, in which case it is just a single concrete store path with * possibly no known derivation * - * - built, in which case it is a pair of a derivation path and an - * output name. + * - built, in which case it is a pair of a derivation path and some + * output names. */ struct DerivedPath : _DerivedPathRaw { using Raw = _DerivedPathRaw; @@ -91,6 +227,18 @@ struct DerivedPath : _DerivedPathRaw { return static_cast(*this); } + /** + * Get the store path this is ultimately derived from (by realising + * and projecting outputs). + * + * Note that this is *not* a property of the store object being + * referred to, but just of this path --- how we happened to be + * referring to that store object. In other words, this means this + * function breaks "referential transparency". It should therefore + * be used only with great care. + */ + const StorePath & getBaseStorePath() const; + /** * Uses `^` as the separator */ @@ -101,55 +249,43 @@ struct DerivedPath : _DerivedPathRaw { std::string to_string_legacy(const Store & store) const; /** * Uses `^` as the separator + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ - static DerivedPath parse(const Store & store, std::string_view); + static DerivedPath parse( + const Store & store, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); /** * Uses `!` as the separator + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ - static DerivedPath parseLegacy(const Store & store, std::string_view); -}; + static DerivedPath parseLegacy( + const Store & store, + std::string_view, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); -/** - * A built derived path with hints in the form of optional concrete output paths. - * - * See 'BuiltPath' for more an explanation. - */ -struct BuiltPathBuilt { - StorePath drvPath; - std::map outputs; - - nlohmann::json toJSON(ref store) const; - static BuiltPathBuilt parse(const Store & store, std::string_view); - - GENERATE_CMP(BuiltPathBuilt, me->drvPath, me->outputs); -}; - -using _BuiltPathRaw = std::variant< - DerivedPath::Opaque, - BuiltPathBuilt ->; - -/** - * A built path. Similar to a DerivedPath, but enriched with the corresponding - * output path(s). - */ -struct BuiltPath : _BuiltPathRaw { - using Raw = _BuiltPathRaw; - using Raw::Raw; - - using Opaque = DerivedPathOpaque; - using Built = BuiltPathBuilt; - - inline const Raw & raw() const { - return static_cast(*this); - } - - StorePathSet outPaths() const; - RealisedPath::Set toRealisedPaths(Store & store) const; + /** + * Convert a `SingleDerivedPath` to a `DerivedPath`. + */ + static DerivedPath fromSingle(const SingleDerivedPath &); + nlohmann::json toJSON(Store & store) const; }; typedef std::vector DerivedPaths; -typedef std::vector BuiltPaths; +/** + * Used by various parser functions to require experimental features as + * needed. + * + * Somewhat unfortunate this cannot just be an implementation detail for + * this module. + * + * @param xpSettings Stop-gap to avoid globals during unit tests. + */ +void drvRequireExperiment( + const SingleDerivedPath & drv, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); } diff --git a/src/libstore/downstream-placeholder.cc b/src/libstore/downstream-placeholder.cc index 1752738f2..885b3604d 100644 --- a/src/libstore/downstream-placeholder.cc +++ b/src/libstore/downstream-placeholder.cc @@ -11,8 +11,10 @@ std::string DownstreamPlaceholder::render() const DownstreamPlaceholder DownstreamPlaceholder::unknownCaOutput( const StorePath & drvPath, - std::string_view outputName) + std::string_view outputName, + const ExperimentalFeatureSettings & xpSettings) { + xpSettings.require(Xp::CaDerivations); auto drvNameWithExtension = drvPath.name(); auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4); auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName); @@ -36,4 +38,19 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation( }; } +DownstreamPlaceholder DownstreamPlaceholder::fromSingleDerivedPathBuilt( + const SingleDerivedPath::Built & b) +{ + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & o) { + return DownstreamPlaceholder::unknownCaOutput(o.path, b.output); + }, + [&](const SingleDerivedPath::Built & b2) { + return DownstreamPlaceholder::unknownDerivation( + DownstreamPlaceholder::fromSingleDerivedPathBuilt(b2), + b.output); + }, + }, b.drvPath->raw()); +} + } diff --git a/src/libstore/downstream-placeholder.hh b/src/libstore/downstream-placeholder.hh index f0c0dee77..9372dcd58 100644 --- a/src/libstore/downstream-placeholder.hh +++ b/src/libstore/downstream-placeholder.hh @@ -3,6 +3,7 @@ #include "hash.hh" #include "path.hh" +#include "derived-path.hh" namespace nix { @@ -52,10 +53,13 @@ public: * * The derivation itself is known (we have a store path for it), but * the output doesn't yet have a known store path. + * + * @param xpSettings Stop-gap to avoid globals during unit tests. */ static DownstreamPlaceholder unknownCaOutput( const StorePath & drvPath, - std::string_view outputName); + std::string_view outputName, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); /** * Create a placehold for the output of an unknown derivation. @@ -70,6 +74,17 @@ public: const DownstreamPlaceholder & drvPlaceholder, std::string_view outputName, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + + /** + * Convenience constructor that handles both cases (unknown + * content-addressed output and unknown derivation), delegating as + * needed to `unknownCaOutput` and `unknownDerivation`. + * + * Recursively builds up a placeholder from a + * `SingleDerivedPath::Built.drvPath` chain. + */ + static DownstreamPlaceholder fromSingleDerivedPathBuilt( + const SingleDerivedPath::Built & built); }; } diff --git a/src/libstore/gc-store.hh b/src/libstore/gc-store.hh index 2c26c65c4..ab1059fb1 100644 --- a/src/libstore/gc-store.hh +++ b/src/libstore/gc-store.hh @@ -71,19 +71,36 @@ struct GCResults }; +/** + * Mix-in class for \ref Store "stores" which expose a notion of garbage + * collection. + * + * Garbage collection will allow deleting paths which are not + * transitively "rooted". + * + * The notion of GC roots actually not part of this class. + * + * - The base `Store` class has `Store::addTempRoot()` because for a store + * that doesn't support garbage collection at all, a temporary GC root is + * safely implementable as no-op. + * + * @todo actually this is not so good because stores are *views*. + * Some views have only a no-op temp roots even though others to the + * same store allow triggering GC. For instance one can't add a root + * over ssh, but that doesn't prevent someone from gc-ing that store + * accesed via SSH locally). + * + * - The derived `LocalFSStore` class has `LocalFSStore::addPermRoot`, + * which is not part of this class because it relies on the notion of + * an ambient file system. There are stores (`ssh-ng://`, for one), + * that *do* support garbage collection but *don't* expose any file + * system, and `LocalFSStore::addPermRoot` thus does not make sense + * for them. + */ struct GcStore : public virtual Store { inline static std::string operationName = "Garbage collection"; - /** - * Add an indirect root, which is merely a symlink to `path` from - * `/nix/var/nix/gcroots/auto/`. `path` is supposed - * to be a symlink to a store path. The garbage collector will - * automatically remove the indirect root when it finds that - * `path` has disappeared. - */ - virtual void addIndirectRoot(const Path & path) = 0; - /** * Find the roots of the garbage collector. Each root is a pair * `(link, storepath)` where `link` is the path of the symlink diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 20720fb99..26c87391c 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -1,7 +1,6 @@ #include "derivations.hh" #include "globals.hh" #include "local-store.hh" -#include "local-fs-store.hh" #include "finally.hh" #include @@ -50,7 +49,7 @@ void LocalStore::addIndirectRoot(const Path & path) } -Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot) +Path IndirectRootStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot) { Path gcRoot(canonPath(_gcRoot)); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index ec8625020..7009f6bb8 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -193,18 +193,24 @@ public: Setting thisSystem{ this, SYSTEM, "system", R"( - This option specifies the canonical Nix system name of the current - installation, such as `i686-linux` or `x86_64-darwin`. Nix can only - build derivations whose `system` attribute equals the value - specified here. In general, it never makes sense to modify this - value from its default, since you can use it to ‘lie’ about the - platform you are building on (e.g., perform a Mac OS build on a - Linux machine; the result would obviously be wrong). It only makes - sense if the Nix binaries can run on multiple platforms, e.g., - ‘universal binaries’ that run on `x86_64-linux` and `i686-linux`. + The system type of the current Nix installation. + Nix will only build a given [derivation](@docroot@/language/derivations.md) locally when its `system` attribute equals any of the values specified here or in [`extra-platforms`](#conf-extra-platforms). - It defaults to the canonical Nix system name detected by `configure` - at build time. + The default value is set when Nix itself is compiled for the system it will run on. + The following system types are widely used, as [Nix is actively supported on these platforms](@docroot@/contributing/hacking.md#platforms): + + - `x86_64-linux` + - `x86_64-darwin` + - `i686-linux` + - `aarch64-linux` + - `aarch64-darwin` + - `armv6l-linux` + - `armv7l-linux` + + In general, you do not have to modify this setting. + While you can force Nix to run a Darwin-specific `builder` executable on a Linux machine, the result would obviously be wrong. + + This value is available in the Nix language as [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem). )"}; Setting maxSilentTime{ @@ -524,6 +530,24 @@ public: Setting sandboxFallback{this, true, "sandbox-fallback", "Whether to disable sandboxing when the kernel doesn't allow it."}; + Setting requireDropSupplementaryGroups{this, getuid() == 0, "require-drop-supplementary-groups", + R"( + Following the principle of least privilege, + Nix will attempt to drop supplementary groups when building with sandboxing. + + However this can fail under some circumstances. + For example, if the user lacks the `CAP_SETGID` capability. + Search `setgroups(2)` for `EPERM` to find more detailed information on this. + + If you encounter such a failure, setting this option to `false` will let you ignore it and continue. + But before doing so, you should consider the security implications carefully. + Not dropping supplementary groups means the build sandbox will be less restricted than intended. + + This option defaults to `true` when the user is root + (since `root` usually has permissions to call setgroups) + and `false` otherwise. + )"}; + #if __linux__ Setting sandboxShmSize{ this, "50%", "sandbox-dev-shm-size", @@ -652,18 +676,20 @@ public: getDefaultExtraPlatforms(), "extra-platforms", R"( - Platforms other than the native one which this machine is capable of - building for. This can be useful for supporting additional - architectures on compatible machines: i686-linux can be built on - x86\_64-linux machines (and the default for this setting reflects - this); armv7 is backwards-compatible with armv6 and armv5tel; some - aarch64 machines can also natively run 32-bit ARM code; and - qemu-user may be used to support non-native platforms (though this - may be slow and buggy). Most values for this are not enabled by - default because build systems will often misdetect the target - platform and generate incompatible code, so you may wish to - cross-check the results of using this option against proper - natively-built versions of your derivations. + System types of executables that can be run on this machine. + + Nix will only build a given [derivation](@docroot@/language/derivations.md) locally when its `system` attribute equals any of the values specified here or in the [`system` option](#conf-system). + + Setting this can be useful to build derivations locally on compatible machines: + - `i686-linux` executables can be run on `x86_64-linux` machines (set by default) + - `x86_64-darwin` executables can be run on macOS `aarch64-darwin` with Rosetta 2 (set by default where applicable) + - `armv6` and `armv5tel` executables can be run on `armv7` + - some `aarch64` machines can also natively run 32-bit ARM code + - `qemu-user` may be used to support non-native platforms (though this + may be slow and buggy) + + Build systems will usually detect the target platform to be the current physical system and therefore produce machine code incompatible with what may be intended in the derivation. + You should design your derivation's `builder` accordingly and cross-check the results when using this option against natively-built versions of your derivation. )", {}, false}; Setting systemFeatures{ @@ -992,7 +1018,7 @@ public: | `~/.nix-defexpr` | `$XDG_STATE_HOME/nix/defexpr` | | `~/.nix-channels` | `$XDG_STATE_HOME/nix/channels` | - If you already have Nix installed and are using [profiles](@docroot@/package-management/profiles.md) or [channels](@docroot@/package-management/channels.md), you should migrate manually when you enable this option. + If you already have Nix installed and are using [profiles](@docroot@/package-management/profiles.md) or [channels](@docroot@/command-ref/nix-channel.md), you should migrate manually when you enable this option. If `$XDG_STATE_HOME` is not set, use `$HOME/.local/state/nix` instead of `$XDG_STATE_HOME/nix`. This can be achieved with the following shell commands: diff --git a/src/libstore/indirect-root-store.hh b/src/libstore/indirect-root-store.hh new file mode 100644 index 000000000..59e45af45 --- /dev/null +++ b/src/libstore/indirect-root-store.hh @@ -0,0 +1,48 @@ +#pragma once +///@file + +#include "local-fs-store.hh" + +namespace nix { + +/** + * Mix-in class for implementing permanent roots as a pair of a direct + * (strong) reference and indirect weak reference to the first + * reference. + * + * See methods for details on the operations it represents. + */ +struct IndirectRootStore : public virtual LocalFSStore +{ + inline static std::string operationName = "Indirect GC roots registration"; + + /** + * Implementation of `LocalFSStore::addPermRoot` where the permanent + * root is a pair of + * + * - The user-facing symlink which all implementations must create + * + * - An additional weak reference known as the "indirect root" that + * points to that symlink. + * + * The garbage collector will automatically remove the indirect root + * when it finds that the symlink has disappeared. + * + * The implementation of this method is concrete, but it delegates + * to `addIndirectRoot()` which is abstract. + */ + Path addPermRoot(const StorePath & storePath, const Path & gcRoot) override final; + + /** + * Add an indirect root, which is a weak reference to the + * user-facing symlink created by `addPermRoot()`. + * + * @param path user-facing and user-controlled symlink to a store + * path. + * + * The form this weak-reference takes is implementation-specific. + */ + virtual void addIndirectRoot(const Path & path) = 0; +}; + +} diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index fa17d606d..78b05031a 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -358,6 +358,9 @@ public: [&](const StorePath & drvPath) { throw Error("wanted to fetch '%s' but the legacy ssh protocol doesn't support merely substituting drv files via the build paths command. It would build them instead. Try using ssh-ng://", printStorePath(drvPath)); }, + [&](std::monostate) { + throw Error("wanted build derivation that is itself a build product, but the legacy ssh protocol doesn't support that. Try using ssh-ng://"); + }, }, sOrDrvPath); } conn->to << ss; diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 2ee2ef0c8..488109501 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -40,6 +40,7 @@ class LocalFSStore : public virtual LocalFSStoreConfig, public virtual LogStore { public: + inline static std::string operationName = "Local Filesystem Store"; const static std::string drvsLogDir; @@ -49,9 +50,20 @@ public: ref getFSAccessor() override; /** - * Register a permanent GC root. + * Creates symlink from the `gcRoot` to the `storePath` and + * registers the `gcRoot` as a permanent GC root. The `gcRoot` + * symlink lives outside the store and is created and owned by the + * user. + * + * @param gcRoot The location of the symlink. + * + * @param storePath The store object being rooted. The symlink will + * point to `toRealPath(store.printStorePath(storePath))`. + * + * How the permanent GC root corresponding to this symlink is + * managed is implementation-specific. */ - Path addPermRoot(const StorePath & storePath, const Path & gcRoot); + virtual Path addPermRoot(const StorePath & storePath, const Path & gcRoot) = 0; virtual Path getRealStoreDir() { return realStoreDir; } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e69460e6c..17b4ecc73 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1022,10 +1022,9 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path) std::map> -LocalStore::queryPartialDerivationOutputMap(const StorePath & path_) +LocalStore::queryStaticPartialDerivationOutputMap(const StorePath & path) { - auto path = path_; - auto outputs = retrySQLite>>([&]() { + return retrySQLite>>([&]() { auto state(_state.lock()); std::map> outputs; uint64_t drvId; @@ -1037,21 +1036,6 @@ LocalStore::queryPartialDerivationOutputMap(const StorePath & path_) return outputs; }); - - if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) - return outputs; - - auto drv = readInvalidDerivation(path); - auto drvHashes = staticOutputHashes(*this, drv); - for (auto& [outputName, hash] : drvHashes) { - auto realisation = queryRealisation(DrvOutput{hash, outputName}); - if (realisation) - outputs.insert_or_assign(outputName, realisation->outPath); - else - outputs.insert({outputName, std::nullopt}); - } - - return outputs; } std::optional LocalStore::queryPathFromHashPart(const std::string & hashPart) @@ -1212,6 +1196,15 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (checkSigs && pathInfoIsUntrusted(info)) throw Error("cannot add path '%s' because it lacks a signature by a trusted key", printStorePath(info.path)); + /* In case we are not interested in reading the NAR: discard it. */ + bool narRead = false; + Finally cleanup = [&]() { + if (!narRead) { + ParseSink sink; + parseDump(sink, source); + } + }; + addTempRoot(info.path); if (repair || !isValidPath(info.path)) { @@ -1236,6 +1229,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, TeeSource wrapperSource { source, hashSink }; + narRead = true; restorePath(realPath, wrapperSource); auto hashResult = hashSink.finish(); @@ -1249,27 +1243,17 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, printStorePath(info.path), info.narSize, hashResult.second); if (info.ca) { - if (auto foHash = std::get_if(&info.ca->raw)) { - auto actualFoHash = hashCAPath( - foHash->method, - foHash->hash.type, - info.path - ); - if (foHash->hash != actualFoHash.hash) { - throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", - printStorePath(info.path), - foHash->hash.to_string(Base32, true), - actualFoHash.hash.to_string(Base32, true)); - } - } - if (auto textHash = std::get_if(&info.ca->raw)) { - auto actualTextHash = hashString(htSHA256, readFile(realPath)); - if (textHash->hash != actualTextHash) { - throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", - printStorePath(info.path), - textHash->hash.to_string(Base32, true), - actualTextHash.to_string(Base32, true)); - } + auto & specified = *info.ca; + auto actualHash = hashCAPath( + specified.method, + specified.hash.type, + info.path + ); + if (specified.hash != actualHash.hash) { + throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", + printStorePath(info.path), + specified.hash.to_string(Base32, true), + actualHash.hash.to_string(Base32, true)); } } @@ -1349,10 +1333,8 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name auto [hash, size] = hashSink->finish(); ContentAddressWithReferences desc = FixedOutputInfo { - .hash = { - .method = method, - .hash = hash, - }, + .method = method, + .hash = hash, .references = { .others = references, // caller is not capable of creating a self-reference, because this is content-addressed without modulus @@ -1428,8 +1410,8 @@ StorePath LocalStore::addTextToStore( { auto hash = hashString(htSHA256, s); auto dstPath = makeTextPath(name, TextInfo { - { .hash = hash }, - references, + .hash = hash, + .references = references, }); addTempRoot(dstPath); @@ -1459,7 +1441,10 @@ StorePath LocalStore::addTextToStore( ValidPathInfo info { dstPath, narHash }; info.narSize = sink.s.size(); info.references = references; - info.ca = TextHash { .hash = hash }; + info.ca = { + .method = TextIngestionMethod {}, + .hash = hash, + }; registerValidPath(info); } @@ -1524,17 +1509,33 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) auto fdGCLock = openGCLock(); FdLock gcLock(fdGCLock.get(), ltRead, true, "waiting for the big garbage collector lock..."); - StringSet store; - for (auto & i : readDirectory(realStoreDir)) store.insert(i.name); - - /* Check whether all valid paths actually exist. */ - printInfo("checking path existence..."); - StorePathSet validPaths; - PathSet done; - for (auto & i : queryAllValidPaths()) - verifyPath(printStorePath(i), store, done, validPaths, repair, errors); + { + StorePathSet storePathsInStoreDir; + /* Why aren't we using `queryAllValidPaths`? Because that would + tell us about all the paths than the database knows about. Here we + want to know about all the store paths in the store directory, + regardless of what the database thinks. + + We will end up cross-referencing these two sources of truth (the + database and the filesystem) in the loop below, in order to catch + invalid states. + */ + for (auto & i : readDirectory(realStoreDir)) { + try { + storePathsInStoreDir.insert({i.name}); + } catch (BadStorePath &) { } + } + + /* Check whether all valid paths actually exist. */ + printInfo("checking path existence..."); + + StorePathSet done; + + for (auto & i : queryAllValidPaths()) + verifyPath(i, storePathsInStoreDir, done, validPaths, repair, errors); + } /* Optionally, check the content hashes (slow). */ if (checkContents) { @@ -1620,32 +1621,27 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) } -void LocalStore::verifyPath(const Path & pathS, const StringSet & store, - PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors) +void LocalStore::verifyPath(const StorePath & path, const StorePathSet & storePathsInStoreDir, + StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors) { checkInterrupt(); - if (!done.insert(pathS).second) return; + if (!done.insert(path).second) return; - if (!isStorePath(pathS)) { - printError("path '%s' is not in the Nix store", pathS); - return; - } - - auto path = parseStorePath(pathS); - - if (!store.count(std::string(path.to_string()))) { + if (!storePathsInStoreDir.count(path)) { /* Check any referrers first. If we can invalidate them first, then we can invalidate this path as well. */ bool canInvalidate = true; StorePathSet referrers; queryReferrers(path, referrers); for (auto & i : referrers) if (i != path) { - verifyPath(printStorePath(i), store, done, validPaths, repair, errors); + verifyPath(i, storePathsInStoreDir, done, validPaths, repair, errors); if (validPaths.count(i)) canInvalidate = false; } + auto pathS = printStorePath(path); + if (canInvalidate) { printInfo("path '%s' disappeared, removing from database...", pathS); auto state(_state.lock()); @@ -1856,33 +1852,39 @@ void LocalStore::queryRealisationUncached(const DrvOutput & id, } } -FixedOutputHash LocalStore::hashCAPath( - const FileIngestionMethod & method, const HashType & hashType, +ContentAddress LocalStore::hashCAPath( + const ContentAddressMethod & method, const HashType & hashType, const StorePath & path) { return hashCAPath(method, hashType, Store::toRealPath(path), path.hashPart()); } -FixedOutputHash LocalStore::hashCAPath( - const FileIngestionMethod & method, +ContentAddress LocalStore::hashCAPath( + const ContentAddressMethod & method, const HashType & hashType, const Path & path, const std::string_view pathHash ) { HashModuloSink caSink ( hashType, std::string(pathHash) ); - switch (method) { - case FileIngestionMethod::Recursive: - dumpPath(path, caSink); - break; - case FileIngestionMethod::Flat: - readFile(path, caSink); - break; - } - auto hash = caSink.finish().first; - return FixedOutputHash{ + std::visit(overloaded { + [&](const TextIngestionMethod &) { + readFile(path, caSink); + }, + [&](const FileIngestionMethod & m2) { + switch (m2) { + case FileIngestionMethod::Recursive: + dumpPath(path, caSink); + break; + case FileIngestionMethod::Flat: + readFile(path, caSink); + break; + } + }, + }, method.raw); + return ContentAddress { .method = method, - .hash = hash, + .hash = caSink.finish().first, }; } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 8a3b0b43f..e97195f5b 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -5,8 +5,7 @@ #include "pathlocks.hh" #include "store-api.hh" -#include "local-fs-store.hh" -#include "gc-store.hh" +#include "indirect-root-store.hh" #include "sync.hh" #include "util.hh" @@ -68,7 +67,9 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig std::string doc() override; }; -class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore, public virtual GcStore +class LocalStore : public virtual LocalStoreConfig + , public virtual IndirectRootStore + , public virtual GcStore { private: @@ -165,7 +166,7 @@ public: StorePathSet queryValidDerivers(const StorePath & path) override; - std::map> queryPartialDerivationOutputMap(const StorePath & path) override; + std::map> queryStaticPartialDerivationOutputMap(const StorePath & path) override; std::optional queryPathFromHashPart(const std::string & hashPart) override; @@ -209,6 +210,12 @@ private: public: + /** + * Implementation of IndirectRootStore::addIndirectRoot(). + * + * The weak reference merely is a symlink to `path' from + * /nix/var/nix/gcroots/auto/. + */ void addIndirectRoot(const Path & path) override; private: @@ -307,8 +314,8 @@ private: */ void invalidatePathChecked(const StorePath & path); - void verifyPath(const Path & path, const StringSet & store, - PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors); + void verifyPath(const StorePath & path, const StorePathSet & store, + StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors); std::shared_ptr queryPathInfoInternal(State & state, const StorePath & path); @@ -345,13 +352,13 @@ private: void signRealisation(Realisation &); // XXX: Make a generic `Store` method - FixedOutputHash hashCAPath( - const FileIngestionMethod & method, + ContentAddress hashCAPath( + const ContentAddressMethod & method, const HashType & hashType, const StorePath & path); - FixedOutputHash hashCAPath( - const FileIngestionMethod & method, + ContentAddress hashCAPath( + const ContentAddressMethod & method, const HashType & hashType, const Path & path, const std::string_view pathHash diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 626a22480..253609ed2 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -52,10 +52,8 @@ std::map makeContentAddressed( dstStore, path.name(), FixedOutputInfo { - .hash = { - .method = FileIngestionMethod::Recursive, - .hash = narModuloHash, - }, + .method = FileIngestionMethod::Recursive, + .hash = narModuloHash, .references = std::move(refs), }, Hash::dummy, diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 50336c779..185d61c15 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -132,7 +132,7 @@ void Store::queryMissing(const std::vector & targets, } for (auto & i : drv.inputDrvs) - pool.enqueue(std::bind(doPath, DerivedPath::Built { i.first, i.second })); + pool.enqueue(std::bind(doPath, DerivedPath::Built { makeConstantStorePathRef(i.first), i.second })); }; auto checkOutput = [&]( @@ -176,10 +176,18 @@ void Store::queryMissing(const std::vector & targets, std::visit(overloaded { [&](const DerivedPath::Built & bfd) { - if (!isValidPath(bfd.drvPath)) { + auto drvPathP = std::get_if(&*bfd.drvPath); + if (!drvPathP) { + // TODO make work in this case. + warn("Ignoring dynamic derivation %s while querying missing paths; not yet implemented", bfd.drvPath->to_string(*this)); + return; + } + auto & drvPath = drvPathP->path; + + if (!isValidPath(drvPath)) { // FIXME: we could try to substitute the derivation. auto state(state_.lock()); - state->unknown.insert(bfd.drvPath); + state->unknown.insert(drvPath); return; } @@ -187,7 +195,7 @@ void Store::queryMissing(const std::vector & targets, /* true for regular derivations, and CA derivations for which we have a trust mapping for all wanted outputs. */ auto knownOutputPaths = true; - for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(bfd.drvPath)) { + for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(drvPath)) { if (!pathOpt) { knownOutputPaths = false; break; @@ -197,15 +205,45 @@ void Store::queryMissing(const std::vector & targets, } if (knownOutputPaths && invalid.empty()) return; - auto drv = make_ref(derivationFromPath(bfd.drvPath)); - ParsedDerivation parsedDrv(StorePath(bfd.drvPath), *drv); + auto drv = make_ref(derivationFromPath(drvPath)); + ParsedDerivation parsedDrv(StorePath(drvPath), *drv); + + if (!knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) { + experimentalFeatureSettings.require(Xp::CaDerivations); + + // If there are unknown output paths, attempt to find if the + // paths are known to substituters through a realisation. + auto outputHashes = staticOutputHashes(*this, *drv); + knownOutputPaths = true; + + for (auto [outputName, hash] : outputHashes) { + if (!bfd.outputs.contains(outputName)) + continue; + + bool found = false; + for (auto &sub : getDefaultSubstituters()) { + auto realisation = sub->queryRealisation({hash, outputName}); + if (!realisation) + continue; + found = true; + if (!isValidPath(realisation->outPath)) + invalid.insert(realisation->outPath); + break; + } + if (!found) { + // Some paths did not have a realisation, this must be built. + knownOutputPaths = false; + break; + } + } + } if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) { auto drvState = make_ref>(DrvState(invalid.size())); for (auto & output : invalid) - pool.enqueue(std::bind(checkOutput, bfd.drvPath, drv, output, drvState)); + pool.enqueue(std::bind(checkOutput, drvPath, drv, output, drvState)); } else - mustBuildDrv(bfd.drvPath, *drv); + mustBuildDrv(drvPath, *drv); }, [&](const DerivedPath::Opaque & bo) { @@ -310,45 +348,91 @@ std::map drvOutputReferences( OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_) { - auto & evalStore = evalStore_ ? *evalStore_ : store; + auto drvPath = resolveDerivedPath(store, *bfd.drvPath, evalStore_); - OutputPathMap outputs; - auto drv = evalStore.readDerivation(bfd.drvPath); - auto outputHashes = staticOutputHashes(store, drv); - auto drvOutputs = drv.outputsAndOptPaths(store); - auto outputNames = std::visit(overloaded { + auto outputsOpt_ = store.queryPartialDerivationOutputMap(drvPath, evalStore_); + + auto outputsOpt = std::visit(overloaded { [&](const OutputsSpec::All &) { - StringSet names; - for (auto & [outputName, _] : drv.outputs) - names.insert(outputName); - return names; + // Keep all outputs + return std::move(outputsOpt_); }, [&](const OutputsSpec::Names & names) { - return static_cast>(names); + // Get just those mentioned by name + std::map> outputsOpt; + for (auto & output : names) { + auto * pOutputPathOpt = get(outputsOpt_, output); + if (!pOutputPathOpt) + throw Error( + "the derivation '%s' doesn't have an output named '%s'", + bfd.drvPath->to_string(store), output); + outputsOpt.insert_or_assign(output, std::move(*pOutputPathOpt)); + } + return outputsOpt; }, }, bfd.outputs.raw()); - for (auto & output : outputNames) { - auto outputHash = get(outputHashes, output); - if (!outputHash) - throw Error( - "the derivation '%s' doesn't have an output named '%s'", - store.printStorePath(bfd.drvPath), output); - if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { - DrvOutput outputId { *outputHash, output }; - auto realisation = store.queryRealisation(outputId); - if (!realisation) - throw MissingRealisation(outputId); - outputs.insert_or_assign(output, realisation->outPath); - } else { - // If ca-derivations isn't enabled, assume that - // the output path is statically known. - auto drvOutput = get(drvOutputs, output); - assert(drvOutput); - assert(drvOutput->second); - outputs.insert_or_assign(output, *drvOutput->second); - } + + OutputPathMap outputs; + for (auto & [outputName, outputPathOpt] : outputsOpt) { + if (!outputPathOpt) + throw MissingRealisation(bfd.drvPath->to_string(store), outputName); + auto & outputPath = *outputPathOpt; + outputs.insert_or_assign(outputName, outputPath); } return outputs; } + +StorePath resolveDerivedPath(Store & store, const SingleDerivedPath & req, Store * evalStore_) +{ + auto & evalStore = evalStore_ ? *evalStore_ : store; + + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) { + return bo.path; + }, + [&](const SingleDerivedPath::Built & bfd) { + auto drvPath = resolveDerivedPath(store, *bfd.drvPath, evalStore_); + auto outputPaths = evalStore.queryPartialDerivationOutputMap(drvPath, evalStore_); + if (outputPaths.count(bfd.output) == 0) + throw Error("derivation '%s' does not have an output named '%s'", + store.printStorePath(drvPath), bfd.output); + auto & optPath = outputPaths.at(bfd.output); + if (!optPath) + throw Error("'%s' does not yet map to a known concrete store path", + bfd.to_string(store)); + return *optPath; + }, + }, req.raw()); +} + + +OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd) +{ + auto drvPath = resolveDerivedPath(store, *bfd.drvPath); + auto outputMap = store.queryDerivationOutputMap(drvPath); + auto outputsLeft = std::visit(overloaded { + [&](const OutputsSpec::All &) { + return StringSet {}; + }, + [&](const OutputsSpec::Names & names) { + return static_cast(names); + }, + }, bfd.outputs.raw()); + for (auto iter = outputMap.begin(); iter != outputMap.end();) { + auto & outputName = iter->first; + if (bfd.outputs.contains(outputName)) { + outputsLeft.erase(outputName); + ++iter; + } else { + iter = outputMap.erase(iter); + } + } + if (!outputsLeft.empty()) + throw Error("derivation '%s' does not have an outputs %s", + store.printStorePath(drvPath), + concatStringsSep(", ", quoteStrings(std::get(bfd.outputs)))); + return outputMap; +} + } diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 981bbfb14..ccb57104f 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -29,14 +29,14 @@ std::optional ValidPathInfo::contentAddressWithRef return std::nullopt; return std::visit(overloaded { - [&](const TextHash & th) -> ContentAddressWithReferences { + [&](const TextIngestionMethod &) -> ContentAddressWithReferences { assert(references.count(path) == 0); return TextInfo { - .hash = th, + .hash = ca->hash, .references = references, }; }, - [&](const FixedOutputHash & foh) -> ContentAddressWithReferences { + [&](const FileIngestionMethod & m2) -> ContentAddressWithReferences { auto refs = references; bool hasSelfReference = false; if (refs.count(path)) { @@ -44,14 +44,15 @@ std::optional ValidPathInfo::contentAddressWithRef refs.erase(path); } return FixedOutputInfo { - .hash = foh, + .method = m2, + .hash = ca->hash, .references = { .others = std::move(refs), .self = hasSelfReference, }, }; }, - }, ca->raw); + }, ca->method.raw); } bool ValidPathInfo::isContentAddressed(const Store & store) const @@ -110,13 +111,19 @@ ValidPathInfo::ValidPathInfo( std::visit(overloaded { [this](TextInfo && ti) { this->references = std::move(ti.references); - this->ca = std::move((TextHash &&) ti); + this->ca = ContentAddress { + .method = TextIngestionMethod {}, + .hash = std::move(ti.hash), + }; }, [this](FixedOutputInfo && foi) { this->references = std::move(foi.references.others); if (foi.references.self) this->references.insert(path); - this->ca = std::move((FixedOutputHash &&) foi); + this->ca = ContentAddress { + .method = std::move(foi.method), + .hash = std::move(foi.hash), + }; }, }, std::move(ca).raw); } diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index 869b490ad..81f360f3a 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -16,10 +16,16 @@ std::string StorePathWithOutputs::to_string(const Store & store) const DerivedPath StorePathWithOutputs::toDerivedPath() const { if (!outputs.empty()) { - return DerivedPath::Built { path, OutputsSpec::Names { outputs } }; + return DerivedPath::Built { + .drvPath = makeConstantStorePathRef(path), + .outputs = OutputsSpec::Names { outputs }, + }; } else if (path.isDerivation()) { assert(outputs.empty()); - return DerivedPath::Built { path, OutputsSpec::All { } }; + return DerivedPath::Built { + .drvPath = makeConstantStorePathRef(path), + .outputs = OutputsSpec::All { }, + }; } else { return DerivedPath::Opaque { path }; } @@ -34,29 +40,36 @@ std::vector toDerivedPaths(const std::vector } -std::variant StorePathWithOutputs::tryFromDerivedPath(const DerivedPath & p) +StorePathWithOutputs::ParseResult StorePathWithOutputs::tryFromDerivedPath(const DerivedPath & p) { return std::visit(overloaded { - [&](const DerivedPath::Opaque & bo) -> std::variant { + [&](const DerivedPath::Opaque & bo) -> StorePathWithOutputs::ParseResult { if (bo.path.isDerivation()) { // drv path gets interpreted as "build", not "get drv file itself" return bo.path; } return StorePathWithOutputs { bo.path }; }, - [&](const DerivedPath::Built & bfd) -> std::variant { - return StorePathWithOutputs { - .path = bfd.drvPath, - // Use legacy encoding of wildcard as empty set - .outputs = std::visit(overloaded { - [&](const OutputsSpec::All &) -> StringSet { - return {}; - }, - [&](const OutputsSpec::Names & outputs) { - return static_cast(outputs); - }, - }, bfd.outputs.raw()), - }; + [&](const DerivedPath::Built & bfd) -> StorePathWithOutputs::ParseResult { + return std::visit(overloaded { + [&](const SingleDerivedPath::Opaque & bo) -> StorePathWithOutputs::ParseResult { + return StorePathWithOutputs { + .path = bo.path, + // Use legacy encoding of wildcard as empty set + .outputs = std::visit(overloaded { + [&](const OutputsSpec::All &) -> StringSet { + return {}; + }, + [&](const OutputsSpec::Names & outputs) { + return static_cast(outputs); + }, + }, bfd.outputs.raw()), + }; + }, + [&](const SingleDerivedPath::Built &) -> StorePathWithOutputs::ParseResult { + return std::monostate {}; + }, + }, bfd.drvPath->raw()); }, }, p.raw()); } diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index d75850868..57e03252d 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -23,7 +23,9 @@ struct StorePathWithOutputs DerivedPath toDerivedPath() const; - static std::variant tryFromDerivedPath(const DerivedPath &); + typedef std::variant ParseResult; + + static StorePathWithOutputs::ParseResult tryFromDerivedPath(const DerivedPath &); }; std::vector toDerivedPaths(const std::vector); diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index 2a093c128..0548b30c1 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -5,6 +5,7 @@ #include "hash.hh" #include "path.hh" +#include "derived-path.hh" #include #include "comparator.hh" #include "crypto.hh" @@ -143,9 +144,13 @@ class MissingRealisation : public Error { public: MissingRealisation(DrvOutput & outputId) - : Error( "cannot operate on an output of the " + : MissingRealisation(outputId.outputName, outputId.strHash()) + {} + MissingRealisation(std::string_view drv, std::string outputName) + : Error( "cannot operate on output '%s' of the " "unbuilt derivation '%s'", - outputId.to_string()) + outputName, + drv) {} }; diff --git a/src/libstore/remote-store-connection.hh b/src/libstore/remote-store-connection.hh index d32d91a60..ce4740a9c 100644 --- a/src/libstore/remote-store-connection.hh +++ b/src/libstore/remote-store-connection.hh @@ -1,5 +1,6 @@ #include "remote-store.hh" #include "worker-protocol.hh" +#include "pool.hh" namespace nix { @@ -94,4 +95,34 @@ struct RemoteStore::Connection std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0, bool flush = true); }; +/** + * A wrapper around Pool::Handle that marks + * the connection as bad (causing it to be closed) if a non-daemon + * exception is thrown before the handle is closed. Such an exception + * causes a deviation from the expected protocol and therefore a + * desynchronization between the client and daemon. + */ +struct RemoteStore::ConnectionHandle +{ + Pool::Handle handle; + bool daemonException = false; + + ConnectionHandle(Pool::Handle && handle) + : handle(std::move(handle)) + { } + + ConnectionHandle(ConnectionHandle && h) + : handle(std::move(h.handle)) + { } + + ~ConnectionHandle(); + + RemoteStore::Connection & operator * () { return *handle; } + RemoteStore::Connection * operator -> () { return &*handle; } + + void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true); + + void withFramedSink(std::function fun); +}; + } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 1e2104e1f..58f72beb9 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -159,49 +159,25 @@ void RemoteStore::setOptions(Connection & conn) } -/* A wrapper around Pool::Handle that marks - the connection as bad (causing it to be closed) if a non-daemon - exception is thrown before the handle is closed. Such an exception - causes a deviation from the expected protocol and therefore a - desynchronization between the client and daemon. */ -struct ConnectionHandle +RemoteStore::ConnectionHandle::~ConnectionHandle() { - Pool::Handle handle; - bool daemonException = false; - - ConnectionHandle(Pool::Handle && handle) - : handle(std::move(handle)) - { } - - ConnectionHandle(ConnectionHandle && h) - : handle(std::move(h.handle)) - { } - - ~ConnectionHandle() - { - if (!daemonException && std::uncaught_exceptions()) { - handle.markBad(); - debug("closing daemon connection because of an exception"); - } + if (!daemonException && std::uncaught_exceptions()) { + handle.markBad(); + debug("closing daemon connection because of an exception"); } +} - RemoteStore::Connection * operator -> () { return &*handle; } - RemoteStore::Connection & operator * () { return *handle; } - - void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true) - { - auto ex = handle->processStderr(sink, source, flush); - if (ex) { - daemonException = true; - std::rethrow_exception(ex); - } +void RemoteStore::ConnectionHandle::processStderr(Sink * sink, Source * source, bool flush) +{ + auto ex = handle->processStderr(sink, source, flush); + if (ex) { + daemonException = true; + std::rethrow_exception(ex); } - - void withFramedSink(std::function fun); -}; +} -ConnectionHandle RemoteStore::getConnection() +RemoteStore::ConnectionHandle RemoteStore::getConnection() { return ConnectionHandle(connections->get()); } @@ -378,27 +354,36 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path) } -std::map> RemoteStore::queryPartialDerivationOutputMap(const StorePath & path) +std::map> RemoteStore::queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore_) { if (GET_PROTOCOL_MINOR(getProtocol()) >= 0x16) { - auto conn(getConnection()); - conn->to << WorkerProto::Op::QueryDerivationOutputMap << printStorePath(path); - conn.processStderr(); - return WorkerProto::Serialise>>::read(*this, *conn); + if (!evalStore_) { + auto conn(getConnection()); + conn->to << WorkerProto::Op::QueryDerivationOutputMap << printStorePath(path); + conn.processStderr(); + return WorkerProto::Serialise>>::read(*this, *conn); + } else { + auto & evalStore = *evalStore_; + auto outputs = evalStore.queryStaticPartialDerivationOutputMap(path); + // union with the first branch overriding the statically-known ones + // when non-`std::nullopt`. + for (auto && [outputName, optPath] : queryPartialDerivationOutputMap(path, nullptr)) { + if (optPath) + outputs.insert_or_assign(std::move(outputName), std::move(optPath)); + else + outputs.insert({std::move(outputName), std::nullopt}); + } + return outputs; + } } else { + auto & evalStore = evalStore_ ? *evalStore_ : *this; // Fallback for old daemon versions. // For floating-CA derivations (and their co-dependencies) this is an // under-approximation as it only returns the paths that can be inferred // from the derivation itself (and not the ones that are known because // the have been built), but as old stores don't handle floating-CA // derivations this shouldn't matter - auto derivation = readDerivation(path); - auto outputsWithOptPaths = derivation.outputsAndOptPaths(*this); - std::map> ret; - for (auto & [outputName, outputAndPath] : outputsWithOptPaths) { - ret.emplace(outputName, outputAndPath.second); - } - return ret; + return evalStore.queryStaticPartialDerivationOutputMap(path); } } @@ -671,6 +656,9 @@ static void writeDerivedPaths(RemoteStore & store, RemoteStore::Connection & con GET_PROTOCOL_MAJOR(conn.daemonVersion), GET_PROTOCOL_MINOR(conn.daemonVersion)); }, + [&](std::monostate) { + throw Error("wanted to build a derivation that is itself a build product, but the legacy 'ssh://' protocol doesn't support that. Try using 'ssh-ng://'"); + }, }, sOrDrvPath); } conn.to << ss; @@ -685,9 +673,16 @@ void RemoteStore::copyDrvsFromEvalStore( /* The remote doesn't have a way to access evalStore, so copy the .drvs. */ RealisedPath::Set drvPaths2; - for (auto & i : paths) - if (auto p = std::get_if(&i)) - drvPaths2.insert(p->drvPath); + for (const auto & i : paths) { + std::visit(overloaded { + [&](const DerivedPath::Opaque & bp) { + // Do nothing, path is hopefully there already + }, + [&](const DerivedPath::Built & bp) { + drvPaths2.insert(bp.drvPath->getBaseStorePath()); + }, + }, i.raw()); + } copyClosure(*evalStore, *this, drvPaths2); } } @@ -757,7 +752,8 @@ std::vector RemoteStore::buildPathsWithResults( }; OutputPathMap outputs; - auto drv = evalStore->readDerivation(bfd.drvPath); + auto drvPath = resolveDerivedPath(*evalStore, *bfd.drvPath); + auto drv = evalStore->readDerivation(drvPath); const auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive auto built = resolveDerivedPath(*this, bfd, &*evalStore); for (auto & [output, outputPath] : built) { @@ -765,7 +761,7 @@ std::vector RemoteStore::buildPathsWithResults( if (!outputHash) throw Error( "the derivation '%s' doesn't have an output named '%s'", - printStorePath(bfd.drvPath), output); + printStorePath(drvPath), output); auto outputId = DrvOutput{ *outputHash, output }; if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { auto realisation = @@ -837,15 +833,6 @@ void RemoteStore::addTempRoot(const StorePath & path) } -void RemoteStore::addIndirectRoot(const Path & path) -{ - auto conn(getConnection()); - conn->to << WorkerProto::Op::AddIndirectRoot << path; - conn.processStderr(); - readInt(conn->from); -} - - Roots RemoteStore::findRoots(bool censor) { auto conn(getConnection()); @@ -1090,7 +1077,7 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * return nullptr; } -void ConnectionHandle::withFramedSink(std::function fun) +void RemoteStore::ConnectionHandle::withFramedSink(std::function fun) { (*this)->to.flush(); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index cb7a71acf..a1ae82a0f 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -17,7 +17,6 @@ class Pid; struct FdSink; struct FdSource; template class Pool; -struct ConnectionHandle; struct RemoteStoreConfig : virtual StoreConfig { @@ -63,7 +62,7 @@ public: StorePathSet queryDerivationOutputs(const StorePath & path) override; - std::map> queryPartialDerivationOutputMap(const StorePath & path) override; + std::map> queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore = nullptr) override; std::optional queryPathFromHashPart(const std::string & hashPart) override; StorePathSet querySubstitutablePaths(const StorePathSet & paths) override; @@ -127,8 +126,6 @@ public: void addTempRoot(const StorePath & path) override; - void addIndirectRoot(const Path & path) override; - Roots findRoots(bool censor) override; void collectGarbage(const GCOptions & options, GCResults & results) override; @@ -182,6 +179,8 @@ protected: void setOptions() override; + struct ConnectionHandle; + ConnectionHandle getConnection(); friend struct ConnectionHandle; @@ -199,5 +198,4 @@ private: std::shared_ptr evalStore); }; - } diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 0200076c0..9c6c42ef4 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -1,5 +1,6 @@ #include "ssh-store-config.hh" #include "store-api.hh" +#include "local-fs-store.hh" #include "remote-store.hh" #include "remote-store-connection.hh" #include "remote-fs-accessor.hh" @@ -61,7 +62,7 @@ public: std::optional getBuildLogExact(const StorePath & path) override { unsupported("getBuildLogExact"); } -private: +protected: struct Connection : RemoteStore::Connection { @@ -93,9 +94,12 @@ private: ref SSHStore::openConnection() { auto conn = make_ref(); - conn->sshConn = master.startCommand( - fmt("%s --stdio", remoteProgram) - + (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get()))); + + std::string command = remoteProgram + " --stdio"; + if (remoteStore.get() != "") + command += " --store " + shellEscape(remoteStore.get()); + + conn->sshConn = master.startCommand(command); conn->to = FdSink(conn->sshConn->in.get()); conn->from = FdSource(conn->sshConn->out.get()); return conn; diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index fae99d75b..da32f1b79 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -42,7 +42,10 @@ void SSHMaster::addCommonSSHOpts(Strings & args) } bool SSHMaster::isMasterRunning() { - auto res = runProgram(RunOptions {.program = "ssh", .args = {"-O", "check", host}, .mergeStderrToStdout = true}); + Strings args = {"-O", "check", host}; + addCommonSSHOpts(args); + + auto res = runProgram(RunOptions {.program = "ssh", .args = args, .mergeStderrToStdout = true}); return res.first == 0; } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 5bee1af9f..28689e100 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -184,15 +184,15 @@ static std::string makeType( StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { - if (info.hash.hash.type == htSHA256 && info.hash.method == FileIngestionMethod::Recursive) { - return makeStorePath(makeType(*this, "source", info.references), info.hash.hash, name); + if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) { + return makeStorePath(makeType(*this, "source", info.references), info.hash, name); } else { assert(info.references.size() == 0); return makeStorePath("output:out", hashString(htSHA256, "fixed:out:" - + makeFileIngestionPrefix(info.hash.method) - + info.hash.hash.to_string(Base16, true) + ":"), + + makeFileIngestionPrefix(info.method) + + info.hash.to_string(Base16, true) + ":"), name); } } @@ -200,13 +200,13 @@ StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInf StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const { - assert(info.hash.hash.type == htSHA256); + assert(info.hash.type == htSHA256); return makeStorePath( makeType(*this, "text", StoreReferences { .others = info.references, .self = false, }), - info.hash.hash, + info.hash, name); } @@ -232,10 +232,8 @@ std::pair Store::computeStorePathForPath(std::string_view name, ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); FixedOutputInfo caInfo { - .hash = { - .method = method, - .hash = h, - }, + .method = method, + .hash = h, .references = {}, }; return std::make_pair(makeFixedOutputPath(name, caInfo), h); @@ -248,8 +246,8 @@ StorePath Store::computeStorePathForText( const StorePathSet & references) const { return makeTextPath(name, TextInfo { - { .hash = hashString(htSHA256, s) }, - references, + .hash = hashString(htSHA256, s), + .references = references, }); } @@ -441,10 +439,8 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, *this, name, FixedOutputInfo { - .hash = { - .method = method, - .hash = hash, - }, + .method = method, + .hash = hash, .references = {}, }, narHash, @@ -496,22 +492,50 @@ bool Store::PathInfoCacheValue::isKnownNow() return std::chrono::steady_clock::now() < time_point + ttl; } -std::map> Store::queryPartialDerivationOutputMap(const StorePath & path) +std::map> Store::queryStaticPartialDerivationOutputMap(const StorePath & path) { std::map> outputs; auto drv = readInvalidDerivation(path); - for (auto& [outputName, output] : drv.outputsAndOptPaths(*this)) { + for (auto & [outputName, output] : drv.outputsAndOptPaths(*this)) { outputs.emplace(outputName, output.second); } return outputs; } +std::map> Store::queryPartialDerivationOutputMap( + const StorePath & path, + Store * evalStore_) +{ + auto & evalStore = evalStore_ ? *evalStore_ : *this; + + auto outputs = evalStore.queryStaticPartialDerivationOutputMap(path); + + if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) + return outputs; + + auto drv = evalStore.readInvalidDerivation(path); + auto drvHashes = staticOutputHashes(*this, drv); + for (auto & [outputName, hash] : drvHashes) { + auto realisation = queryRealisation(DrvOutput{hash, outputName}); + if (realisation) { + outputs.insert_or_assign(outputName, realisation->outPath); + } else { + // queryStaticPartialDerivationOutputMap is not guaranteed + // to return std::nullopt for outputs which are not + // statically known. + outputs.insert({outputName, std::nullopt}); + } + } + + return outputs; +} + OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) { auto resp = queryPartialDerivationOutputMap(path); OutputPathMap result; for (auto & [outName, optOutPath] : resp) { if (!optOutPath) - throw Error("output '%s' of derivation '%s' has no store path mapped to it", outName, printStorePath(path)); + throw MissingRealisation(printStorePath(path), outName); result.insert_or_assign(outName, *optOutPath); } return result; @@ -1472,6 +1496,7 @@ ref openStore(const std::string & uri_, if (implem.uriSchemes.count(parsedUri.scheme)) { auto store = implem.create(parsedUri.scheme, baseURI, params); if (store) { + experimentalFeatureSettings.require(store->experimentalFeature()); store->init(); store->warnUnknownSettings(); return ref(store); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 14a862eef..da1a3eefb 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -99,6 +99,8 @@ typedef std::map> StorePathCAMap; struct StoreConfig : public Config { + typedef std::map Params; + using Config::Config; StoreConfig() = delete; @@ -107,13 +109,28 @@ struct StoreConfig : public Config virtual ~StoreConfig() { } + /** + * The name of this type of store. + */ virtual const std::string name() = 0; + /** + * Documentation for this type of store. + */ virtual std::string doc() { return ""; } + /** + * An experimental feature this type store is gated, if it is to be + * experimental. + */ + virtual std::optional experimentalFeature() const + { + return std::nullopt; + } + const PathSetting storeDir_{this, settings.nixStore, "store", R"( @@ -153,10 +170,6 @@ struct StoreConfig : public Config class Store : public std::enable_shared_from_this, public virtual StoreConfig { -public: - - typedef std::map Params; - protected: struct PathInfoCacheValue { @@ -425,7 +438,20 @@ public: * derivation. All outputs are mentioned so ones mising the mapping * are mapped to `std::nullopt`. */ - virtual std::map> queryPartialDerivationOutputMap(const StorePath & path); + virtual std::map> queryPartialDerivationOutputMap( + const StorePath & path, + Store * evalStore = nullptr); + + /** + * Like `queryPartialDerivationOutputMap` but only considers + * statically known output paths (i.e. those that can be gotten from + * the derivation itself. + * + * Just a helper function for implementing + * `queryPartialDerivationOutputMap`. + */ + virtual std::map> queryStaticPartialDerivationOutputMap( + const StorePath & path); /** * Query the mapping outputName=>outputPath for the given derivation. @@ -919,6 +945,7 @@ void removeTempRoots(); * Resolve the derived path completely, failing if any derivation output * is unknown. */ +StorePath resolveDerivedPath(Store &, const SingleDerivedPath &, Store * evalStore = nullptr); OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * evalStore = nullptr); diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc index 6328ad370..0e28c1f08 100644 --- a/src/libstore/tests/derivation.cc +++ b/src/libstore/tests/derivation.cc @@ -81,7 +81,7 @@ TEST_JSON(DerivationTest, caFixedFlat, "path": "/nix/store/rhcg9h16sqvlbpsa6dqm57sbr2al6nzg-drv-name-output-name" })", (DerivationOutput::CAFixed { - .ca = FixedOutputHash { + .ca = { .method = FileIngestionMethod::Flat, .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), }, @@ -95,7 +95,7 @@ TEST_JSON(DerivationTest, caFixedNAR, "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" })", (DerivationOutput::CAFixed { - .ca = FixedOutputHash { + .ca = { .method = FileIngestionMethod::Recursive, .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), }, @@ -109,7 +109,7 @@ TEST_JSON(DynDerivationTest, caFixedText, "path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name" })", (DerivationOutput::CAFixed { - .ca = TextHash { + .ca = { .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), }, }), diff --git a/src/libstore/tests/derived-path.cc b/src/libstore/tests/derived-path.cc index 160443ec1..d6549f66f 100644 --- a/src/libstore/tests/derived-path.cc +++ b/src/libstore/tests/derived-path.cc @@ -17,14 +17,34 @@ Gen Arbitrary::arbitrary() }); } +Gen Arbitrary::arbitrary() +{ + return gen::just(SingleDerivedPath::Built { + .drvPath = make_ref(*gen::arbitrary()), + .output = (*gen::arbitrary()).name, + }); +} + Gen Arbitrary::arbitrary() { return gen::just(DerivedPath::Built { - .drvPath = *gen::arbitrary(), + .drvPath = make_ref(*gen::arbitrary()), .outputs = *gen::arbitrary(), }); } +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, std::variant_size_v)) { + case 0: + return gen::just(*gen::arbitrary()); + case 1: + return gen::just(*gen::arbitrary()); + default: + assert(false); + } +} + Gen Arbitrary::arbitrary() { switch (*gen::inRange(0, std::variant_size_v)) { @@ -45,12 +65,69 @@ class DerivedPathTest : public LibStoreTest { }; -// FIXME: `RC_GTEST_FIXTURE_PROP` isn't calling `SetUpTestSuite` because it is -// no a real fixture. -// -// See https://github.com/emil-e/rapidcheck/blob/master/doc/gtest.md#rc_gtest_fixture_propfixture-name-args -TEST_F(DerivedPathTest, force_init) -{ +/** + * Round trip (string <-> data structure) test for + * `DerivedPath::Opaque`. + */ +TEST_F(DerivedPathTest, opaque) { + std::string_view opaque = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x"; + auto elem = DerivedPath::parse(*store, opaque); + auto * p = std::get_if(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->path, store->parseStorePath(opaque)); + ASSERT_EQ(elem.to_string(*store), opaque); +} + +/** + * Round trip (string <-> data structure) test for a simpler + * `DerivedPath::Built`. + */ +TEST_F(DerivedPathTest, built_opaque) { + std::string_view built = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv^bar,foo"; + auto elem = DerivedPath::parse(*store, built); + auto * p = std::get_if(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->outputs, ((OutputsSpec) OutputsSpec::Names { "foo", "bar" })); + ASSERT_EQ(*p->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque { + .path = store->parseStorePath(built.substr(0, 49)), + })); + ASSERT_EQ(elem.to_string(*store), built); +} + +/** + * Round trip (string <-> data structure) test for a more complex, + * inductive `DerivedPath::Built`. + */ +TEST_F(DerivedPathTest, built_built) { + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations"); + + std::string_view built = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv^foo^bar,baz"; + auto elem = DerivedPath::parse(*store, built, mockXpSettings); + auto * p = std::get_if(&elem); + ASSERT_TRUE(p); + ASSERT_EQ(p->outputs, ((OutputsSpec) OutputsSpec::Names { "bar", "baz" })); + auto * drvPath = std::get_if(&*p->drvPath); + ASSERT_TRUE(drvPath); + ASSERT_EQ(drvPath->output, "foo"); + ASSERT_EQ(*drvPath->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque { + .path = store->parseStorePath(built.substr(0, 49)), + })); + ASSERT_EQ(elem.to_string(*store), built); +} + +/** + * Without the right experimental features enabled, we cannot parse a + * complex inductive derived path. + */ +TEST_F(DerivedPathTest, built_built_xp) { + ASSERT_THROW( + DerivedPath::parse(*store, "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv^foo^bar,baz"), + MissingExperimentalFeature); } RC_GTEST_FIXTURE_PROP( diff --git a/src/libstore/tests/derived-path.hh b/src/libstore/tests/derived-path.hh index 506f3ccb1..98d61f228 100644 --- a/src/libstore/tests/derived-path.hh +++ b/src/libstore/tests/derived-path.hh @@ -12,8 +12,18 @@ namespace rc { using namespace nix; template<> -struct Arbitrary { - static Gen arbitrary(); +struct Arbitrary { + static Gen arbitrary(); +}; + +template<> +struct Arbitrary { + static Gen arbitrary(); +}; + +template<> +struct Arbitrary { + static Gen arbitrary(); }; template<> diff --git a/src/libstore/tests/downstream-placeholder.cc b/src/libstore/tests/downstream-placeholder.cc index ec3e1000f..fd29530ac 100644 --- a/src/libstore/tests/downstream-placeholder.cc +++ b/src/libstore/tests/downstream-placeholder.cc @@ -5,17 +5,24 @@ namespace nix { TEST(DownstreamPlaceholder, unknownCaOutput) { + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "ca-derivations"); + ASSERT_EQ( DownstreamPlaceholder::unknownCaOutput( StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" }, - "out").render(), + "out", + mockXpSettings).render(), "/0c6rn30q4frawknapgwq386zq358m8r6msvywcvc89n6m5p2dgbz"); } TEST(DownstreamPlaceholder, unknownDerivation) { /** - * We set these in tests rather than the regular globals so we don't have - * to worry about race conditions if the tests run concurrently. + * Same reason as above */ ExperimentalFeatureSettings mockXpSettings; mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations"); @@ -24,7 +31,8 @@ TEST(DownstreamPlaceholder, unknownDerivation) { DownstreamPlaceholder::unknownDerivation( DownstreamPlaceholder::unknownCaOutput( StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv.drv" }, - "out"), + "out", + mockXpSettings), "out", mockXpSettings).render(), "/0gn6agqxjyyalf0dpihgyf49xq5hqxgw100f0wydnj6yqrhqsb3w"); diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 69dae2da5..99589f8b2 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -1,4 +1,5 @@ #include "uds-remote-store.hh" +#include "worker-protocol.hh" #include #include @@ -77,6 +78,15 @@ ref UDSRemoteStore::openConnection() } +void UDSRemoteStore::addIndirectRoot(const Path & path) +{ + auto conn(getConnection()); + conn->to << WorkerProto::Op::AddIndirectRoot << path; + conn.processStderr(); + readInt(conn->from); +} + + static RegisterStoreImplementation regUDSRemoteStore; } diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index 2bd6517fa..cdb28a001 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -3,13 +3,13 @@ #include "remote-store.hh" #include "remote-store-connection.hh" -#include "local-fs-store.hh" +#include "indirect-root-store.hh" namespace nix { struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig { - UDSRemoteStoreConfig(const Store::Params & params) + UDSRemoteStoreConfig(const Params & params) : StoreConfig(params) , LocalFSStoreConfig(params) , RemoteStoreConfig(params) @@ -21,7 +21,9 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon std::string doc() override; }; -class UDSRemoteStore : public virtual UDSRemoteStoreConfig, public virtual LocalFSStore, public virtual RemoteStore +class UDSRemoteStore : public virtual UDSRemoteStoreConfig + , public virtual IndirectRootStore + , public virtual RemoteStore { public: @@ -39,6 +41,16 @@ public: void narFromPath(const StorePath & path, Sink & sink) override { LocalFSStore::narFromPath(path, sink); } + /** + * Implementation of `IndirectRootStore::addIndirectRoot()` which + * delegates to the remote store. + * + * The idea is that the client makes the direct symlink, so it is + * owned managed by the client's user account, and the server makes + * the indirect symlink. + */ + void addIndirectRoot(const Path & path) override; + private: struct Connection : RemoteStore::Connection diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh index 9f661c5c3..7982fdc5e 100644 --- a/src/libutil/comparator.hh +++ b/src/libutil/comparator.hh @@ -1,6 +1,22 @@ #pragma once ///@file +/** + * Declare comparison methods without defining them. + */ +#define DECLARE_ONE_CMP(COMPARATOR, MY_TYPE) \ + bool operator COMPARATOR(const MY_TYPE & other) const; +#define DECLARE_EQUAL(my_type) \ + DECLARE_ONE_CMP(==, my_type) +#define DECLARE_LEQ(my_type) \ + DECLARE_ONE_CMP(<, my_type) +#define DECLARE_NEQ(my_type) \ + DECLARE_ONE_CMP(!=, my_type) +#define DECLARE_CMP(my_type) \ + DECLARE_EQUAL(my_type) \ + DECLARE_LEQ(my_type) \ + DECLARE_NEQ(my_type) + /** * Awful hacky generation of the comparison operators by doing a lexicographic * comparison between the choosen fields. diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 7c4112d32..782331283 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -12,7 +12,7 @@ struct ExperimentalFeatureDetails std::string_view description; }; -constexpr std::array xpFeatureDetails = {{ +constexpr std::array xpFeatureDetails = {{ { .tag = Xp::CaDerivations, .name = "ca-derivations", @@ -182,15 +182,6 @@ constexpr std::array xpFeatureDetails = {{ the [`use-cgroups`](#conf-use-cgroups) setting for details. )", }, - { - .tag = Xp::DiscardReferences, - .name = "discard-references", - .description = R"( - Allow the use of the [`unsafeDiscardReferences`](@docroot@/language/advanced-attributes.html#adv-attr-unsafeDiscardReferences) attribute in derivations - that use [structured attributes](@docroot@/language/advanced-attributes.html#adv-attr-structuredAttrs). This disables scanning of outputs for - runtime dependencies. - )", - }, { .tag = Xp::DaemonTrustOverride, .name = "daemon-trust-override", diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index faf2e9398..add592ae6 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -27,7 +27,6 @@ enum struct ExperimentalFeature ReplFlake, AutoAllocateUids, Cgroups, - DiscardReferences, DaemonTrustOverride, DynamicDerivations, ParseTomlTimestamps, diff --git a/src/libutil/json-utils.cc b/src/libutil/json-utils.cc index d7220e71d..61cef743d 100644 --- a/src/libutil/json-utils.cc +++ b/src/libutil/json-utils.cc @@ -1,4 +1,5 @@ #include "json-utils.hh" +#include "error.hh" namespace nix { @@ -16,4 +17,27 @@ nlohmann::json * get(nlohmann::json & map, const std::string & key) return &*i; } +const nlohmann::json & valueAt( + const nlohmann::json & map, + const std::string & key) +{ + if (!map.contains(key)) + throw Error("Expected JSON object to contain key '%s' but it doesn't", key); + + return map[key]; +} + +const nlohmann::json & ensureType( + const nlohmann::json & value, + nlohmann::json::value_type expectedType + ) +{ + if (value.type() != expectedType) + throw Error( + "Expected JSON value to be of type '%s' but it is of type '%s'", + nlohmann::json(expectedType).type_name(), + value.type_name()); + + return value; +} } diff --git a/src/libutil/json-utils.hh b/src/libutil/json-utils.hh index 5e63c1af4..77c63595c 100644 --- a/src/libutil/json-utils.hh +++ b/src/libutil/json-utils.hh @@ -10,6 +10,28 @@ const nlohmann::json * get(const nlohmann::json & map, const std::string & key); nlohmann::json * get(nlohmann::json & map, const std::string & key); +/** + * Get the value of a json object at a key safely, failing + * with a Nix Error if the key does not exist. + * + * Use instead of nlohmann::json::at() to avoid ugly exceptions. + * + * _Does not check whether `map` is an object_, use `ensureType` for that. + */ +const nlohmann::json & valueAt( + const nlohmann::json & map, + const std::string & key); + +/** + * Ensure the type of a json object is what you expect, failing + * with a Nix Error if it isn't. + * + * Use before type conversions and element access to avoid ugly exceptions. + */ +const nlohmann::json & ensureType( + const nlohmann::json & value, + nlohmann::json::value_type expectedType); + /** * For `adl_serializer>` below, we need to track what * types are not already using `null`. Only for them can we use `null` diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 6510df8f0..66f319c3e 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -393,7 +393,7 @@ static void main_nix_build(int argc, char * * argv) auto bashDrv = drv->requireDrvPath(); pathsToBuild.push_back(DerivedPath::Built { - .drvPath = bashDrv, + .drvPath = makeConstantStorePathRef(bashDrv), .outputs = OutputsSpec::Names {"out"}, }); pathsToCopy.insert(bashDrv); @@ -417,7 +417,7 @@ static void main_nix_build(int argc, char * * argv) })) { pathsToBuild.push_back(DerivedPath::Built { - .drvPath = inputDrv, + .drvPath = makeConstantStorePathRef(inputDrv), .outputs = OutputsSpec::Names { inputOutputs }, }); pathsToCopy.insert(inputDrv); @@ -590,7 +590,10 @@ static void main_nix_build(int argc, char * * argv) if (outputName == "") throw Error("derivation '%s' lacks an 'outputName' attribute", store->printStorePath(drvPath)); - pathsToBuild.push_back(DerivedPath::Built{drvPath, OutputsSpec::Names{outputName}}); + pathsToBuild.push_back(DerivedPath::Built{ + .drvPath = makeConstantStorePathRef(drvPath), + .outputs = OutputsSpec::Names{outputName}, + }); pathsToBuildOrdered.push_back({drvPath, {outputName}}); drvsToCopy.insert(drvPath); diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index c1c8edd1d..95f401441 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -5,6 +5,7 @@ #include "store-api.hh" #include "legacy.hh" #include "fetchers.hh" +#include "eval-settings.hh" // for defexpr #include "util.hh" #include @@ -165,7 +166,7 @@ static int main_nix_channel(int argc, char ** argv) // Figure out the name of the `.nix-channels' file to use auto home = getHome(); channelsList = settings.useXDGBaseDirectories ? createNixStateDir() + "/channels" : home + "/.nix-channels"; - nixDefExpr = settings.useXDGBaseDirectories ? createNixStateDir() + "/defexpr" : home + "/.nix-defexpr"; + nixDefExpr = getNixDefExpr(); // Figure out the name of the channels profile. profile = profilesDir() + "/channels"; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 91b073b49..b112e8cb3 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -15,6 +15,7 @@ #include "value-to-json.hh" #include "xml-writer.hh" #include "legacy.hh" +#include "eval-settings.hh" // for defexpr #include #include @@ -481,7 +482,7 @@ static void printMissing(EvalState & state, DrvInfos & elems) for (auto & i : elems) if (auto drvPath = i.queryDrvPath()) targets.push_back(DerivedPath::Built{ - .drvPath = *drvPath, + .drvPath = makeConstantStorePathRef(*drvPath), .outputs = OutputsSpec::All { }, }); else @@ -759,7 +760,7 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) std::vector paths { drvPath ? (DerivedPath) (DerivedPath::Built { - .drvPath = *drvPath, + .drvPath = makeConstantStorePathRef(*drvPath), .outputs = OutputsSpec::All { }, }) : (DerivedPath) (DerivedPath::Opaque { @@ -1399,7 +1400,7 @@ static int main_nix_env(int argc, char * * argv) globals.instSource.type = srcUnknown; globals.instSource.systemFilter = "*"; - Path nixExprPath = settings.useXDGBaseDirectories ? createNixStateDir() + "/defexpr" : getHome() + "/.nix-defexpr"; + Path nixExprPath = getNixDefExpr(); if (!pathExists(nixExprPath)) { try { diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index caa0248f1..94956df66 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -220,10 +220,8 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) std::string name = *i++; cout << fmt("%s\n", store->printStorePath(store->makeFixedOutputPath(name, FixedOutputInfo { - .hash = { - .method = method, - .hash = Hash::parseAny(hash, hashAlgo), - }, + .method = method, + .hash = Hash::parseAny(hash, hashAlgo), .references = {}, }))); } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 16e48a39b..39e5cc99d 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -45,10 +45,8 @@ struct CmdAddToStore : MixDryRun, StoreCommand *store, std::move(*namePart), FixedOutputInfo { - .hash = { - .method = std::move(ingestionMethod), - .hash = std::move(hash), - }, + .method = std::move(ingestionMethod), + .hash = std::move(hash), .references = {}, }, narHash, diff --git a/src/nix/app.cc b/src/nix/app.cc index e678b54f0..16a921194 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -22,11 +22,13 @@ StringPairs resolveRewrites( StringPairs res; for (auto & dep : dependencies) if (auto drvDep = std::get_if(&dep.path)) - for (auto & [ outputName, outputPath ] : drvDep->outputs) - res.emplace( - DownstreamPlaceholder::unknownCaOutput(drvDep->drvPath, outputName).render(), - store.printStorePath(outputPath) - ); + if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) + for (auto & [ outputName, outputPath ] : drvDep->outputs) + res.emplace( + DownstreamPlaceholder::unknownCaOutput( + drvDep->drvPath->outPath(), outputName).render(), + store.printStorePath(outputPath) + ); return res; } @@ -64,7 +66,7 @@ UnresolvedApp InstallableValue::toApp(EvalState & state) [&](const NixStringContextElem::DrvDeep & d) -> DerivedPath { /* We want all outputs of the drv */ return DerivedPath::Built { - .drvPath = d.drvPath, + .drvPath = makeConstantStorePathRef(d.drvPath), .outputs = OutputsSpec::All {}, }; }, @@ -105,7 +107,7 @@ UnresolvedApp InstallableValue::toApp(EvalState & state) auto program = outPath + "/bin/" + mainProgram; return UnresolvedApp { App { .context = { DerivedPath::Built { - .drvPath = drvPath, + .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::Names { outputName }, } }, .program = program, diff --git a/src/nix/build.cc b/src/nix/build.cc index ad1842a4e..479100186 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -9,18 +9,18 @@ using namespace nix; -nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, ref store) +static nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, Store & store) { auto res = nlohmann::json::array(); for (auto & t : paths) { - std::visit([&res, store](const auto & t) { + std::visit([&](const auto & t) { res.push_back(t.toJSON(store)); }, t.raw()); } return res; } -nlohmann::json builtPathsWithResultToJSON(const std::vector & buildables, ref store) +static nlohmann::json builtPathsWithResultToJSON(const std::vector & buildables, const Store & store) { auto res = nlohmann::json::array(); for (auto & b : buildables) { @@ -125,7 +125,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile printMissing(store, pathsToBuild, lvlError); if (json) - logger->cout("%s", derivedPathsToJSON(pathsToBuild, store).dump()); + logger->cout("%s", derivedPathsToJSON(pathsToBuild, *store).dump()); return; } @@ -136,7 +136,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile installables, repair ? bmRepair : buildMode); - if (json) logger->cout("%s", builtPathsWithResultToJSON(buildables, store).dump()); + if (json) logger->cout("%s", builtPathsWithResultToJSON(buildables, *store).dump()); if (outLink != "") if (auto store2 = store.dynamic_pointer_cast()) diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index bcc00d490..5a80f0308 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -109,7 +109,7 @@ struct CmdBundle : InstallableValueCommand store->buildPaths({ DerivedPath::Built { - .drvPath = drvPath, + .drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All { }, }, }); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 195eeaa21..c033804e4 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -235,7 +235,7 @@ static StorePath getDerivationEnvironment(ref store, ref evalStore /* Build the derivation. */ store->buildPaths( { DerivedPath::Built { - .drvPath = shellDrvPath, + .drvPath = makeConstantStorePathRef(shellDrvPath), .outputs = OutputsSpec::All { }, }}, bmNormal, evalStore); diff --git a/src/nix/flake.cc b/src/nix/flake.cc index b5f5d0cac..83b74c8ca 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -4,6 +4,7 @@ #include "shared.hh" #include "eval.hh" #include "eval-inline.hh" +#include "eval-settings.hh" #include "flake/flake.hh" #include "get-drvs.hh" #include "store-api.hh" @@ -543,9 +544,9 @@ struct CmdFlakeCheck : FlakeCommand *attr2.value, attr2.pos); if (drvPath && attr_name == settings.thisSystem.get()) { drvPaths.push_back(DerivedPath::Built { - .drvPath = *drvPath, - .outputs = OutputsSpec::All { }, - }); + .drvPath = makeConstantStorePathRef(*drvPath), + .outputs = OutputsSpec::All { }, + }); } } } diff --git a/src/nix/log.cc b/src/nix/log.cc index aaf829764..9a9bd30f9 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -33,6 +33,17 @@ struct CmdLog : InstallableCommand auto b = installable->toDerivedPath(); + // For compat with CLI today, TODO revisit + auto oneUp = std::visit(overloaded { + [&](const DerivedPath::Opaque & bo) { + return make_ref(bo); + }, + [&](const DerivedPath::Built & bfd) { + return bfd.drvPath; + }, + }, b.path.raw()); + auto path = resolveDerivedPath(*store, *oneUp); + RunPager pager; for (auto & sub : subs) { auto * logSubP = dynamic_cast(&*sub); @@ -42,14 +53,7 @@ struct CmdLog : InstallableCommand } auto & logSub = *logSubP; - auto log = std::visit(overloaded { - [&](const DerivedPath::Opaque & bo) { - return logSub.getBuildLog(bo.path); - }, - [&](const DerivedPath::Built & bfd) { - return logSub.getBuildLog(bfd.drvPath); - }, - }, b.path.raw()); + auto log = logSub.getBuildLog(path); if (!log) continue; stopProgressBar(); printInfo("got build log for '%s' from '%s'", installable->what(), logSub.getUri()); diff --git a/src/nix/main.cc b/src/nix/main.cc index 650c79d14..c5a9c8b33 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -3,6 +3,7 @@ #include "command.hh" #include "common-args.hh" #include "eval.hh" +#include "eval-settings.hh" #include "globals.hh" #include "legacy.hh" #include "shared.hh" @@ -179,8 +180,10 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs for (auto & implem : *Implementations::registered) { auto storeConfig = implem.getConfig(); auto storeName = storeConfig->name(); - stores[storeName]["doc"] = storeConfig->doc(); - stores[storeName]["settings"] = storeConfig->toJSON(); + auto & j = stores[storeName]; + j["doc"] = storeConfig->doc(); + j["settings"] = storeConfig->toJSON(); + j["experimentalFeature"] = storeConfig->experimentalFeature(); } res["stores"] = std::move(stores); diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 3b2e225f6..b67d381ca 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -71,10 +71,8 @@ std::tuple prefetchFile( if (expectedHash) { hashType = expectedHash->type; storePath = store->makeFixedOutputPath(*name, FixedOutputInfo { - .hash = { - .method = ingestionMethod, - .hash = *expectedHash, - }, + .method = ingestionMethod, + .hash = *expectedHash, .references = {}, }); if (store->isValidPath(*storePath)) @@ -127,7 +125,7 @@ std::tuple prefetchFile( auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash); storePath = info.path; assert(info.ca); - hash = info.ca->getHash(); + hash = info.ca->hash; } return {storePath.value(), hash.value()}; diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 300d075cd..8f4725eb0 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -255,10 +255,8 @@ struct ProfileManifest *store, "profile", FixedOutputInfo { - .hash = { - .method = FileIngestionMethod::Recursive, - .hash = narHash, - }, + .method = FileIngestionMethod::Recursive, + .hash = narHash, .references = { .others = std::move(references), // profiles never refer to themselves diff --git a/src/nix/repl.cc b/src/nix/repl.cc index bb14f3f99..9677c1b48 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -1,4 +1,5 @@ #include "eval.hh" +#include "eval-settings.hh" #include "globals.hh" #include "command.hh" #include "installable-value.hh" diff --git a/src/nix/search.cc b/src/nix/search.cc index bf68e6657..7acd0132d 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -2,6 +2,7 @@ #include "globals.hh" #include "eval.hh" #include "eval-inline.hh" +#include "eval-settings.hh" #include "names.hh" #include "get-drvs.hh" #include "common-args.hh" diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 3997c98bf..d238456db 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -3,6 +3,7 @@ #include "store-api.hh" #include "filetransfer.hh" #include "eval.hh" +#include "eval-settings.hh" #include "attr-path.hh" #include "names.hh" #include "progress-bar.hh" @@ -146,7 +147,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand auto req = FileTransferRequest(storePathsUrl); auto res = getFileTransfer()->download(req); - auto state = std::make_unique(Strings(), store); + auto state = std::make_unique(SearchPath{}, store); auto v = state->allocValue(); state->eval(state->parseExprFromString(res.data, state->rootPath(CanonPath("/no-such-path"))), *v); Bindings & bindings(*state->allocBindings(0)); diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index a3a9dc698..592de773c 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -239,7 +239,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions if (pos != std::string::npos) { size_t margin = 32; auto pos2 = pos >= margin ? pos - margin : 0; - hits[hash].emplace_back(fmt("%s: …%s…\n", + hits[hash].emplace_back(fmt("%s: …%s…", p2, hilite(filterPrintable( std::string(contents, pos2, pos - pos2 + hash.size() + margin)), @@ -255,7 +255,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions for (auto & hash : hashes) { auto pos = target.find(hash); if (pos != std::string::npos) - hits[hash].emplace_back(fmt("%s -> %s\n", p2, + hits[hash].emplace_back(fmt("%s -> %s", p2, hilite(target, pos, StorePath::HashLen, getColour(hash)))); } } @@ -272,9 +272,9 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions for (auto & hit : hits[hash]) { bool first = hit == *hits[hash].begin(); - std::cout << tailPad - << (first ? (last ? treeLast : treeConn) : (last ? treeNull : treeLine)) - << hit; + logger->cout("%s%s%s", tailPad, + (first ? (last ? treeLast : treeConn) : (last ? treeNull : treeLine)), + hit); if (!all) break; } diff --git a/tests/build-remote.sh b/tests/build-remote.sh index 78e12b477..d2a2132c1 100644 --- a/tests/build-remote.sh +++ b/tests/build-remote.sh @@ -1,6 +1,7 @@ requireSandboxSupport [[ $busybox =~ busybox ]] || skipTest "no busybox" +# Avoid store dir being inside sandbox build-dir unset NIX_STORE_DIR unset NIX_STATE_DIR diff --git a/tests/build.sh b/tests/build.sh index 8ae20f0df..7fbdb0f07 100644 --- a/tests/build.sh +++ b/tests/build.sh @@ -78,7 +78,7 @@ expectStderr 1 nix build --impure --expr 'with (import ./multiple-outputs.nix).e | grepQuiet "has 2 entries in its context. It should only have exactly one entry" nix build --impure --json --expr 'builtins.unsafeDiscardOutputDependency (import ./multiple-outputs.nix).e.a_a.drvPath' --no-link | jq --exit-status ' - (.[0] | .path | match(".*multiple-outputs-e.drv")) + (.[0] | match(".*multiple-outputs-e.drv")) ' # Test building from raw store path to drv not expression. diff --git a/tests/ca/build-cache.sh b/tests/ca/build-cache.sh new file mode 100644 index 000000000..6a4080fec --- /dev/null +++ b/tests/ca/build-cache.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +source common.sh + +# The substituters didn't work prior to this time. +requireDaemonNewerThan "2.18.0pre20230808" + +drv=$(nix-instantiate ./content-addressed.nix -A rootCA --arg seed 1)^out +nix derivation show "$drv" --arg seed 1 + +buildAttr () { + local derivationPath=$1 + local seedValue=$2 + shift; shift + local args=("./content-addressed.nix" "-A" "$derivationPath" --arg seed "$seedValue" "--no-out-link") + args+=("$@") + nix-build "${args[@]}" +} + +copyAttr () { + local derivationPath=$1 + local seedValue=$2 + shift; shift + local args=("-f" "./content-addressed.nix" "$derivationPath" --arg seed "$seedValue") + args+=("$@") + # Note: to copy CA derivations, we need to copy the realisations, which + # currently requires naming the installables, not just the derivation output + # path. + nix copy --to file://$cacheDir "${args[@]}" +} + +testRemoteCacheFor () { + local derivationPath=$1 + clearCache + copyAttr "$derivationPath" 1 + clearStore + # Check nothing gets built. + buildAttr "$derivationPath" 1 --option substituters file://$cacheDir --no-require-sigs |& grepQuietInverse " will be built:" +} + +testRemoteCache () { + testRemoteCacheFor rootCA + testRemoteCacheFor dependentCA + testRemoteCacheFor dependentNonCA + testRemoteCacheFor dependentFixedOutput + testRemoteCacheFor dependentForBuildCA + testRemoteCacheFor dependentForBuildNonCA +} + +clearStore +testRemoteCache \ No newline at end of file diff --git a/tests/ca/build.sh b/tests/ca/build.sh index 7754ad276..e1a8a7625 100644 --- a/tests/ca/build.sh +++ b/tests/ca/build.sh @@ -2,7 +2,7 @@ source common.sh -drv=$(nix-instantiate ./content-addressed.nix -A rootCA --arg seed 1) +drv=$(nix-instantiate ./content-addressed.nix -A rootCA --arg seed 1)^out nix derivation show "$drv" --arg seed 1 buildAttr () { @@ -14,14 +14,6 @@ buildAttr () { nix-build "${args[@]}" } -testRemoteCache () { - clearCache - local outPath=$(buildAttr dependentNonCA 1) - nix copy --to file://$cacheDir $outPath - clearStore - buildAttr dependentNonCA 1 --option substituters file://$cacheDir --no-require-sigs |& grepQuietInverse "building dependent-non-ca" -} - testDeterministicCA () { [[ $(buildAttr rootCA 1) = $(buildAttr rootCA 2) ]] } @@ -66,8 +58,6 @@ testNormalization () { test "$(stat -c %Y $outPath)" -eq 1 } -# Disabled until we have it properly working -# testRemoteCache clearStore testNormalization testDeterministicCA diff --git a/tests/ca/content-addressed.nix b/tests/ca/content-addressed.nix index 81bc4bf5c..2559c562f 100644 --- a/tests/ca/content-addressed.nix +++ b/tests/ca/content-addressed.nix @@ -61,6 +61,24 @@ rec { echo ${rootCA}/non-ca-hello > $out/dep ''; }; + dependentForBuildCA = mkCADerivation { + name = "dependent-for-build-ca"; + buildCommand = '' + echo "Depends on rootCA for building only" + mkdir -p $out + echo ${rootCA} + touch $out + ''; + }; + dependentForBuildNonCA = mkDerivation { + name = "dependent-for-build-non-ca"; + buildCommand = '' + echo "Depends on rootCA for building only" + mkdir -p $out + echo ${rootCA} + touch $out + ''; + }; dependentFixedOutput = mkDerivation { name = "dependent-fixed-output"; outputHashMode = "recursive"; diff --git a/tests/ca/local.mk b/tests/ca/local.mk new file mode 100644 index 000000000..0852e592e --- /dev/null +++ b/tests/ca/local.mk @@ -0,0 +1,28 @@ +ca-tests := \ + $(d)/build-with-garbage-path.sh \ + $(d)/build.sh \ + $(d)/build-cache.sh \ + $(d)/concurrent-builds.sh \ + $(d)/derivation-json.sh \ + $(d)/duplicate-realisation-in-closure.sh \ + $(d)/gc.sh \ + $(d)/import-derivation.sh \ + $(d)/new-build-cmd.sh \ + $(d)/nix-copy.sh \ + $(d)/nix-run.sh \ + $(d)/nix-shell.sh \ + $(d)/post-hook.sh \ + $(d)/recursive.sh \ + $(d)/repl.sh \ + $(d)/selfref-gc.sh \ + $(d)/signatures.sh \ + $(d)/substitute.sh \ + $(d)/why-depends.sh + +install-tests-groups += ca + +clean-files += \ + $(d)/config.nix + +test-deps += \ + tests/ca/config.nix diff --git a/tests/check-refs.sh b/tests/check-refs.sh index 2778e491d..3b587d1e5 100644 --- a/tests/check-refs.sh +++ b/tests/check-refs.sh @@ -42,8 +42,10 @@ nix-build -o $RESULT check-refs.nix -A test7 nix-build -o $RESULT check-refs.nix -A test10 if isDaemonNewer 2.12pre20230103; then - enableFeatures discard-references - restartDaemon + if ! isDaemonNewer 2.16.0; then + enableFeatures discard-references + restartDaemon + fi # test11 should succeed. test11=$(nix-build -o $RESULT check-refs.nix -A test11) diff --git a/tests/common.sh b/tests/common.sh index 8941671d6..7b0922c9f 100644 --- a/tests/common.sh +++ b/tests/common.sh @@ -4,7 +4,7 @@ if [[ -z "${COMMON_SH_SOURCED-}" ]]; then COMMON_SH_SOURCED=1 -source "$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")/common/vars-and-functions.sh" +source "$(readlink -f "$(dirname "${BASH_SOURCE[0]-$0}")")/common/vars-and-functions.sh" if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then startDaemon fi diff --git a/tests/common/vars-and-functions.sh.in b/tests/common/vars-and-functions.sh.in index a9e6c802f..dc7ce13cc 100644 --- a/tests/common/vars-and-functions.sh.in +++ b/tests/common/vars-and-functions.sh.in @@ -4,7 +4,7 @@ if [[ -z "${COMMON_VARS_AND_FUNCTIONS_SH_SOURCED-}" ]]; then COMMON_VARS_AND_FUNCTIONS_SH_SOURCED=1 -export PS4='+(${BASH_SOURCE[0]}:$LINENO) ' +export PS4='+(${BASH_SOURCE[0]-$0}:$LINENO) ' export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default} export NIX_STORE_DIR diff --git a/tests/dependencies.sh b/tests/dependencies.sh index f9da0c6bc..d5cd30396 100644 --- a/tests/dependencies.sh +++ b/tests/dependencies.sh @@ -15,6 +15,9 @@ if test -n "$dot"; then $dot < $TEST_ROOT/graph fi +# Test GraphML graph generation +nix-store -q --graphml "$drvPath" > $TEST_ROOT/graphml + outPath=$(nix-store -rvv "$drvPath") || fail "build failed" # Test Graphviz graph generation. diff --git a/tests/dyn-drv/build-built-drv.sh b/tests/dyn-drv/build-built-drv.sh new file mode 100644 index 000000000..647be9457 --- /dev/null +++ b/tests/dyn-drv/build-built-drv.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +source common.sh + +# In the corresponding nix file, we have two derivations: the first, named `hello`, +# is a normal recursive derivation, while the second, named dependent, has the +# new outputHashMode "text". Note that in "dependent", we don't refer to the +# build output of `hello`, but only to the path of the drv file. For this reason, +# we only need to: +# +# - instantiate `hello` +# - build `producingDrv` +# - check that the path of the output coincides with that of the original derivation + +out1=$(nix build -f ./text-hashed-output.nix hello --no-link) + +clearStore + +drvDep=$(nix-instantiate ./text-hashed-output.nix -A producingDrv) + +expectStderr 1 nix build "${drvDep}^out^out" --no-link | grepQuiet "Building dynamic derivations in one shot is not yet implemented" diff --git a/tests/dyn-drv/local.mk b/tests/dyn-drv/local.mk new file mode 100644 index 000000000..b087ecd1c --- /dev/null +++ b/tests/dyn-drv/local.mk @@ -0,0 +1,12 @@ +dyn-drv-tests := \ + $(d)/text-hashed-output.sh \ + $(d)/recursive-mod-json.sh \ + $(d)/build-built-drv.sh + +install-tests-groups += dyn-drv + +clean-files += \ + $(d)/config.nix + +test-deps += \ + tests/dyn-drv/config.nix diff --git a/tests/fetchTree-file.sh b/tests/fetchTree-file.sh index fe569cfb8..6395c133d 100644 --- a/tests/fetchTree-file.sh +++ b/tests/fetchTree-file.sh @@ -27,6 +27,7 @@ test_file_flake_input () { mkdir inputs echo foo > inputs/test_input_file + echo '{ outputs = { self }: { }; }' > inputs/flake.nix tar cfa test_input.tar.gz inputs cp test_input.tar.gz test_input_no_ext input_tarball_hash="$(nix hash path test_input.tar.gz)" @@ -50,6 +51,9 @@ test_file_flake_input () { url = "file+file://$PWD/test_input.tar.gz"; flake = false; }; + inputs.flake_no_ext = { + url = "file://$PWD/test_input_no_ext"; + }; outputs = { ... }: {}; } EOF @@ -58,7 +62,7 @@ EOF nix eval --file - < $out + ''; + }; + +in + + mkDerivation { + shell = busybox; + name = "hermetic"; + passthru = { inherit input1 input2 input3; }; + buildCommand = + '' + read x < ${input1} + read y < ${input3} + echo "$x $y" > $out + ''; + } diff --git a/tests/lang-test-infra.sh b/tests/lang-test-infra.sh new file mode 100644 index 000000000..30da8977b --- /dev/null +++ b/tests/lang-test-infra.sh @@ -0,0 +1,86 @@ +# Test the function for lang.sh +source common.sh + +source lang/framework.sh + +# We are testing this, so don't want outside world to affect us. +unset _NIX_TEST_ACCEPT + +# We'll only modify this in subshells so we don't need to reset it. +badDiff=0 + +# matches non-empty +echo Hi! > "$TEST_ROOT/got" +cp "$TEST_ROOT/got" "$TEST_ROOT/expected" +( + diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 0 )) +) + +# matches empty, non-existant file is the same as empty file +echo -n > "$TEST_ROOT/got" +( + diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/does-not-exist" + (( "$badDiff" == 0 )) +) + +# doesn't matches non-empty, non-existant file is the same as empty file +echo Hi! > "$TEST_ROOT/got" +( + diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/does-not-exist" + (( "$badDiff" == 1 )) +) + +# doesn't match, `badDiff` set, file unchanged +echo Hi! > "$TEST_ROOT/got" +echo Bye! > "$TEST_ROOT/expected" +( + diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 1 )) +) +[[ "$(echo Bye! )" == $(< "$TEST_ROOT/expected") ]] + +# _NIX_TEST_ACCEPT=1 matches non-empty +echo Hi! > "$TEST_ROOT/got" +cp "$TEST_ROOT/got" "$TEST_ROOT/expected" +( + _NIX_TEST_ACCEPT=1 diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 0 )) +) + +# _NIX_TEST_ACCEPT doesn't match, `badDiff=1` set, file changed (was previously non-empty) +echo Hi! > "$TEST_ROOT/got" +echo Bye! > "$TEST_ROOT/expected" +( + _NIX_TEST_ACCEPT=1 diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 1 )) +) +[[ "$(echo Hi! )" == $(< "$TEST_ROOT/expected") ]] +# second time succeeds +( + diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 0 )) +) + +# _NIX_TEST_ACCEPT matches empty, non-existant file not created +echo -n > "$TEST_ROOT/got" +( + _NIX_TEST_ACCEPT=1 diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/does-not-exists" + (( "$badDiff" == 0 )) +) +[[ ! -f "$TEST_ROOT/does-not-exist" ]] + +# _NIX_TEST_ACCEPT doesn't match, output empty, file deleted +echo -n > "$TEST_ROOT/got" +echo Bye! > "$TEST_ROOT/expected" +badDiff=0 +( + _NIX_TEST_ACCEPT=1 diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 1 )) +) +[[ ! -f "$TEST_ROOT/expected" ]] +# second time succeeds +( + diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" + (( "$badDiff" == 0 )) +) diff --git a/tests/lang.sh b/tests/lang.sh old mode 100644 new mode 100755 index 8170cb39d..75dbbc38e --- a/tests/lang.sh +++ b/tests/lang.sh @@ -1,5 +1,17 @@ source common.sh +set -o pipefail + +source lang/framework.sh + +# specialize function a bit +function diffAndAccept() { + local -r testName="$1" + local -r got="lang/$testName.$2" + local -r expected="lang/$testName.$3" + diffAndAcceptInner "$testName" "$got" "$expected" +} + export TEST_VAR=foo # for eval-okay-getenv.nix export NIX_REMOTE=dummy:// export NIX_STORE_DIR=/nix/store @@ -20,63 +32,115 @@ nix-instantiate --eval -E 'let x = { repeating = x; tracing = builtins.trace x t set +x -fail=0 +badDiff=0 +badExitCode=0 for i in lang/parse-fail-*.nix; do echo "parsing $i (should fail)"; - i=$(basename $i .nix) - if ! expect 1 nix-instantiate --parse - < lang/$i.nix; then + i=$(basename "$i" .nix) + if expectStderr 1 nix-instantiate --parse - < "lang/$i.nix" > "lang/$i.err" + then + diffAndAccept "$i" err err.exp + else echo "FAIL: $i shouldn't parse" - fail=1 + badExitCode=1 fi done for i in lang/parse-okay-*.nix; do echo "parsing $i (should succeed)"; - i=$(basename $i .nix) - if ! expect 0 nix-instantiate --parse - < lang/$i.nix > lang/$i.out; then + i=$(basename "$i" .nix) + if + expect 0 nix-instantiate --parse - < "lang/$i.nix" \ + 1> "lang/$i.out" \ + 2> "lang/$i.err" + then + sed "s!$(pwd)!/pwd!g" "lang/$i.out" "lang/$i.err" + diffAndAccept "$i" out exp + diffAndAccept "$i" err err.exp + else echo "FAIL: $i should parse" - fail=1 + badExitCode=1 fi done for i in lang/eval-fail-*.nix; do echo "evaluating $i (should fail)"; - i=$(basename $i .nix) - if ! expect 1 nix-instantiate --eval lang/$i.nix; then + i=$(basename "$i" .nix) + if + expectStderr 1 nix-instantiate --show-trace "lang/$i.nix" \ + | sed "s!$(pwd)!/pwd!g" > "lang/$i.err" + then + diffAndAccept "$i" err err.exp + else echo "FAIL: $i shouldn't evaluate" - fail=1 + badExitCode=1 fi done for i in lang/eval-okay-*.nix; do echo "evaluating $i (should succeed)"; - i=$(basename $i .nix) + i=$(basename "$i" .nix) - if test -e lang/$i.exp; then - flags= - if test -e lang/$i.flags; then - flags=$(cat lang/$i.flags) - fi - if ! expect 0 env NIX_PATH=lang/dir3:lang/dir4 HOME=/fake-home nix-instantiate $flags --eval --strict lang/$i.nix > lang/$i.out; then + if test -e "lang/$i.exp.xml"; then + if expect 0 nix-instantiate --eval --xml --no-location --strict \ + "lang/$i.nix" > "lang/$i.out.xml" + then + diffAndAccept "$i" out.xml exp.xml + else echo "FAIL: $i should evaluate" - fail=1 - elif ! diff <(< lang/$i.out sed -e "s|$(pwd)|/pwd|g") lang/$i.exp; then - echo "FAIL: evaluation result of $i not as expected" - fail=1 + badExitCode=1 + fi + elif test ! -e "lang/$i.exp-disabled"; then + declare -a flags=() + if test -e "lang/$i.flags"; then + read -r -a flags < "lang/$i.flags" fi - fi - if test -e lang/$i.exp.xml; then - if ! expect 0 nix-instantiate --eval --xml --no-location --strict \ - lang/$i.nix > lang/$i.out.xml; then + if + expect 0 env \ + NIX_PATH=lang/dir3:lang/dir4 \ + HOME=/fake-home \ + nix-instantiate "${flags[@]}" --eval --strict "lang/$i.nix" \ + 1> "lang/$i.out" \ + 2> "lang/$i.err" + then + sed -i "s!$(pwd)!/pwd!g" "lang/$i.out" "lang/$i.err" + diffAndAccept "$i" out exp + diffAndAccept "$i" err err.exp + else echo "FAIL: $i should evaluate" - fail=1 - elif ! cmp -s lang/$i.out.xml lang/$i.exp.xml; then - echo "FAIL: XML evaluation result of $i not as expected" - fail=1 + badExitCode=1 fi fi done -exit $fail +if test -n "${_NIX_TEST_ACCEPT-}"; then + if (( "$badDiff" )); then + echo 'Output did mot match, but accepted output as the persisted expected output.' + echo 'That means the next time the tests are run, they should pass.' + else + echo 'NOTE: Environment variable _NIX_TEST_ACCEPT is defined,' + echo 'indicating the unexpected output should be accepted as the expected output going forward,' + echo 'but no tests had unexpected output so there was no expected output to update.' + fi + if (( "$badExitCode" )); then + exit "$badExitCode" + else + skipTest "regenerating golden masters" + fi +else + if (( "$badDiff" )); then + echo '' + echo 'You can rerun this test with:' + echo '' + echo ' _NIX_TEST_ACCEPT=1 make tests/lang.sh.test' + echo '' + echo 'to regenerate the files containing the expected output,' + echo 'and then view the git diff to decide whether a change is' + echo 'good/intentional or bad/unintentional.' + echo 'If the diff contains arbitrary or impure information,' + echo 'please improve the normalization that the test applies to the output.' + fi + exit $(( "$badExitCode" + "$badDiff" )) +fi diff --git a/tests/lang/empty.exp b/tests/lang/empty.exp new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lang/eval-fail-abort.err.exp b/tests/lang/eval-fail-abort.err.exp new file mode 100644 index 000000000..345232d3f --- /dev/null +++ b/tests/lang/eval-fail-abort.err.exp @@ -0,0 +1,10 @@ +error: + … while calling the 'abort' builtin + + at /pwd/lang/eval-fail-abort.nix:1:14: + + 1| if true then abort "this should fail" else 1 + | ^ + 2| + + error: evaluation aborted with the following error message: 'this should fail' diff --git a/tests/lang/eval-fail-antiquoted-path.err.exp b/tests/lang/eval-fail-antiquoted-path.err.exp new file mode 100644 index 000000000..425deba42 --- /dev/null +++ b/tests/lang/eval-fail-antiquoted-path.err.exp @@ -0,0 +1 @@ +error: getting attributes of path ‘PWD/lang/fnord’: No such file or directory diff --git a/tests/lang/eval-fail-assert.err.exp b/tests/lang/eval-fail-assert.err.exp new file mode 100644 index 000000000..aeecd8167 --- /dev/null +++ b/tests/lang/eval-fail-assert.err.exp @@ -0,0 +1,36 @@ +error: + … while evaluating the attribute 'body' + + at /pwd/lang/eval-fail-assert.nix:4:3: + + 3| + 4| body = x "x"; + | ^ + 5| } + + … from call site + + at /pwd/lang/eval-fail-assert.nix:4:10: + + 3| + 4| body = x "x"; + | ^ + 5| } + + … while calling 'x' + + at /pwd/lang/eval-fail-assert.nix:2:7: + + 1| let { + 2| x = arg: assert arg == "y"; 123; + | ^ + 3| + + error: assertion '(arg == "y")' failed + + at /pwd/lang/eval-fail-assert.nix:2:12: + + 1| let { + 2| x = arg: assert arg == "y"; 123; + | ^ + 3| diff --git a/tests/lang/eval-fail-bad-antiquote-1.err.exp b/tests/lang/eval-fail-bad-antiquote-1.err.exp new file mode 100644 index 000000000..cf94f53bc --- /dev/null +++ b/tests/lang/eval-fail-bad-antiquote-1.err.exp @@ -0,0 +1,10 @@ +error: + … while evaluating a path segment + + at /pwd/lang/eval-fail-bad-antiquote-1.nix:1:2: + + 1| "${x: x}" + | ^ + 2| + + error: cannot coerce a function to a string diff --git a/tests/lang/eval-fail-bad-antiquote-2.err.exp b/tests/lang/eval-fail-bad-antiquote-2.err.exp new file mode 100644 index 000000000..c8fe39d12 --- /dev/null +++ b/tests/lang/eval-fail-bad-antiquote-2.err.exp @@ -0,0 +1 @@ +error: operation 'addToStoreFromDump' is not supported by store 'dummy' diff --git a/tests/lang/eval-fail-bad-antiquote-3.err.exp b/tests/lang/eval-fail-bad-antiquote-3.err.exp new file mode 100644 index 000000000..fbefbc826 --- /dev/null +++ b/tests/lang/eval-fail-bad-antiquote-3.err.exp @@ -0,0 +1,10 @@ +error: + … while evaluating a path segment + + at /pwd/lang/eval-fail-bad-antiquote-3.nix:1:3: + + 1| ''${x: x}'' + | ^ + 2| + + error: cannot coerce a function to a string diff --git a/tests/lang/eval-fail-bad-string-interpolation-1.err.exp b/tests/lang/eval-fail-bad-string-interpolation-1.err.exp new file mode 100644 index 000000000..eb73e9a52 --- /dev/null +++ b/tests/lang/eval-fail-bad-string-interpolation-1.err.exp @@ -0,0 +1,10 @@ +error: + … while evaluating a path segment + + at /pwd/lang/eval-fail-bad-string-interpolation-1.nix:1:2: + + 1| "${x: x}" + | ^ + 2| + + error: cannot coerce a function to a string diff --git a/tests/lang/eval-fail-bad-string-interpolation-2.err.exp b/tests/lang/eval-fail-bad-string-interpolation-2.err.exp new file mode 100644 index 000000000..c8fe39d12 --- /dev/null +++ b/tests/lang/eval-fail-bad-string-interpolation-2.err.exp @@ -0,0 +1 @@ +error: operation 'addToStoreFromDump' is not supported by store 'dummy' diff --git a/tests/lang/eval-fail-bad-string-interpolation-3.err.exp b/tests/lang/eval-fail-bad-string-interpolation-3.err.exp new file mode 100644 index 000000000..ac14f329b --- /dev/null +++ b/tests/lang/eval-fail-bad-string-interpolation-3.err.exp @@ -0,0 +1,10 @@ +error: + … while evaluating a path segment + + at /pwd/lang/eval-fail-bad-string-interpolation-3.nix:1:3: + + 1| ''${x: x}'' + | ^ + 2| + + error: cannot coerce a function to a string diff --git a/tests/lang/eval-fail-blackhole.err.exp b/tests/lang/eval-fail-blackhole.err.exp new file mode 100644 index 000000000..f0618d8ac --- /dev/null +++ b/tests/lang/eval-fail-blackhole.err.exp @@ -0,0 +1,18 @@ +error: + … while evaluating the attribute 'body' + + at /pwd/lang/eval-fail-blackhole.nix:2:3: + + 1| let { + 2| body = x; + | ^ + 3| x = y; + + error: infinite recursion encountered + + at /pwd/lang/eval-fail-blackhole.nix:3:7: + + 2| body = x; + 3| x = y; + | ^ + 4| y = x; diff --git a/tests/lang/eval-fail-deepseq.err.exp b/tests/lang/eval-fail-deepseq.err.exp new file mode 100644 index 000000000..5e204ba73 --- /dev/null +++ b/tests/lang/eval-fail-deepseq.err.exp @@ -0,0 +1,26 @@ +error: + … while calling the 'deepSeq' builtin + + at /pwd/lang/eval-fail-deepseq.nix:1:1: + + 1| builtins.deepSeq { x = abort "foo"; } 456 + | ^ + 2| + + … while evaluating the attribute 'x' + + at /pwd/lang/eval-fail-deepseq.nix:1:20: + + 1| builtins.deepSeq { x = abort "foo"; } 456 + | ^ + 2| + + … while calling the 'abort' builtin + + at /pwd/lang/eval-fail-deepseq.nix:1:24: + + 1| builtins.deepSeq { x = abort "foo"; } 456 + | ^ + 2| + + error: evaluation aborted with the following error message: 'foo' diff --git a/tests/lang/eval-fail-dup-dynamic-attrs.err.exp b/tests/lang/eval-fail-dup-dynamic-attrs.err.exp new file mode 100644 index 000000000..e01f8e6d0 --- /dev/null +++ b/tests/lang/eval-fail-dup-dynamic-attrs.err.exp @@ -0,0 +1,8 @@ +error: dynamic attribute 'b' already defined at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:2:11 + + at /pwd/lang/eval-fail-dup-dynamic-attrs.nix:3:11: + + 2| set = { "${"" + "b"}" = 1; }; + 3| set = { "${"b" + ""}" = 2; }; + | ^ + 4| } diff --git a/tests/lang/eval-fail-dup-dynamic-attrs.nix b/tests/lang/eval-fail-dup-dynamic-attrs.nix new file mode 100644 index 000000000..7ea17f6c8 --- /dev/null +++ b/tests/lang/eval-fail-dup-dynamic-attrs.nix @@ -0,0 +1,4 @@ +{ + set = { "${"" + "b"}" = 1; }; + set = { "${"b" + ""}" = 2; }; +} diff --git a/tests/lang/eval-fail-foldlStrict-strict-op-application.err.exp b/tests/lang/eval-fail-foldlStrict-strict-op-application.err.exp new file mode 100644 index 000000000..0069285fb --- /dev/null +++ b/tests/lang/eval-fail-foldlStrict-strict-op-application.err.exp @@ -0,0 +1,38 @@ +error: + … while calling the 'foldl'' builtin + + at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:2:1: + + 1| # Tests that the result of applying op is forced even if the value is never used + 2| builtins.foldl' + | ^ + 3| (_: f: f null) + + … while calling anonymous lambda + + at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:3:7: + + 2| builtins.foldl' + 3| (_: f: f null) + | ^ + 4| null + + … from call site + + at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:3:10: + + 2| builtins.foldl' + 3| (_: f: f null) + | ^ + 4| null + + … while calling anonymous lambda + + at /pwd/lang/eval-fail-foldlStrict-strict-op-application.nix:5:6: + + 4| null + 5| [ (_: throw "Not the final value, but is still forced!") (_: 23) ] + | ^ + 6| + + error: Not the final value, but is still forced! diff --git a/tests/lang/eval-fail-fromTOML-timestamps.err.exp b/tests/lang/eval-fail-fromTOML-timestamps.err.exp new file mode 100644 index 000000000..f6bd19f5a --- /dev/null +++ b/tests/lang/eval-fail-fromTOML-timestamps.err.exp @@ -0,0 +1,12 @@ +error: + … while calling the 'fromTOML' builtin + + at /pwd/lang/eval-fail-fromTOML-timestamps.nix:1:1: + + 1| builtins.fromTOML '' + | ^ + 2| key = "value" + + error: while parsing a TOML string: Dates and times are not supported + + at «none»:0: (source not available) diff --git a/tests/lang/eval-fail-hashfile-missing.err.exp b/tests/lang/eval-fail-hashfile-missing.err.exp new file mode 100644 index 000000000..8e77dec1e --- /dev/null +++ b/tests/lang/eval-fail-hashfile-missing.err.exp @@ -0,0 +1,19 @@ +error: + … while calling the 'toString' builtin + + at /pwd/lang/eval-fail-hashfile-missing.nix:4:3: + + 3| in + 4| toString (builtins.concatLists (map (hash: map (builtins.hashFile hash) paths) ["md5" "sha1" "sha256" "sha512"])) + | ^ + 5| + + … while evaluating the first argument passed to builtins.toString + + at «none»:0: (source not available) + + … while calling the 'hashFile' builtin + + at «none»:0: (source not available) + + error: opening file '/pwd/lang/this-file-is-definitely-not-there-7392097': No such file or directory diff --git a/tests/lang/eval-fail-list.err.exp b/tests/lang/eval-fail-list.err.exp new file mode 100644 index 000000000..24d682118 --- /dev/null +++ b/tests/lang/eval-fail-list.err.exp @@ -0,0 +1,10 @@ +error: + … while evaluating one of the elements to concatenate + + at /pwd/lang/eval-fail-list.nix:1:2: + + 1| 8++1 + | ^ + 2| + + error: value is an integer while a list was expected diff --git a/tests/lang/eval-fail-list.nix b/tests/lang/eval-fail-list.nix new file mode 100644 index 000000000..fa749f2f7 --- /dev/null +++ b/tests/lang/eval-fail-list.nix @@ -0,0 +1 @@ +8++1 diff --git a/tests/lang/eval-fail-missing-arg.err.exp b/tests/lang/eval-fail-missing-arg.err.exp new file mode 100644 index 000000000..61fabf0d5 --- /dev/null +++ b/tests/lang/eval-fail-missing-arg.err.exp @@ -0,0 +1,16 @@ +error: + … from call site + + at /pwd/lang/eval-fail-missing-arg.nix:1:1: + + 1| ({x, y, z}: x + y + z) {x = "foo"; z = "bar";} + | ^ + 2| + + error: function 'anonymous lambda' called without required argument 'y' + + at /pwd/lang/eval-fail-missing-arg.nix:1:2: + + 1| ({x, y, z}: x + y + z) {x = "foo"; z = "bar";} + | ^ + 2| diff --git a/tests/lang/eval-fail-nonexist-path.err.exp b/tests/lang/eval-fail-nonexist-path.err.exp new file mode 100644 index 000000000..c8fe39d12 --- /dev/null +++ b/tests/lang/eval-fail-nonexist-path.err.exp @@ -0,0 +1 @@ +error: operation 'addToStoreFromDump' is not supported by store 'dummy' diff --git a/tests/lang/eval-fail-path-slash.err.exp b/tests/lang/eval-fail-path-slash.err.exp new file mode 100644 index 000000000..f0011c97f --- /dev/null +++ b/tests/lang/eval-fail-path-slash.err.exp @@ -0,0 +1,8 @@ +error: path has a trailing slash + + at /pwd/lang/eval-fail-path-slash.nix:6:12: + + 5| # and https://nixos.org/nix-dev/2016-June/020829.html + 6| /nix/store/ + | ^ + 7| diff --git a/tests/lang/eval-fail-recursion.err.exp b/tests/lang/eval-fail-recursion.err.exp new file mode 100644 index 000000000..af64133cb --- /dev/null +++ b/tests/lang/eval-fail-recursion.err.exp @@ -0,0 +1,16 @@ +error: + … in the right operand of the update (//) operator + + at /pwd/lang/eval-fail-recursion.nix:1:12: + + 1| let a = {} // a; in a.foo + | ^ + 2| + + error: infinite recursion encountered + + at /pwd/lang/eval-fail-recursion.nix:1:15: + + 1| let a = {} // a; in a.foo + | ^ + 2| diff --git a/tests/lang/eval-fail-recursion.nix b/tests/lang/eval-fail-recursion.nix new file mode 100644 index 000000000..075b5ed06 --- /dev/null +++ b/tests/lang/eval-fail-recursion.nix @@ -0,0 +1 @@ +let a = {} // a; in a.foo diff --git a/tests/lang/eval-fail-remove.err.exp b/tests/lang/eval-fail-remove.err.exp new file mode 100644 index 000000000..e82cdac98 --- /dev/null +++ b/tests/lang/eval-fail-remove.err.exp @@ -0,0 +1,19 @@ +error: + … while evaluating the attribute 'body' + + at /pwd/lang/eval-fail-remove.nix:4:3: + + 3| + 4| body = (removeAttrs attrs ["x"]).x; + | ^ + 5| } + + error: attribute 'x' missing + + at /pwd/lang/eval-fail-remove.nix:4:10: + + 3| + 4| body = (removeAttrs attrs ["x"]).x; + | ^ + 5| } + Did you mean y? diff --git a/tests/lang/eval-fail-scope-5.err.exp b/tests/lang/eval-fail-scope-5.err.exp new file mode 100644 index 000000000..22b6166f8 --- /dev/null +++ b/tests/lang/eval-fail-scope-5.err.exp @@ -0,0 +1,36 @@ +error: + … while evaluating the attribute 'body' + + at /pwd/lang/eval-fail-scope-5.nix:8:3: + + 7| + 8| body = f {}; + | ^ + 9| + + … from call site + + at /pwd/lang/eval-fail-scope-5.nix:8:10: + + 7| + 8| body = f {}; + | ^ + 9| + + … while calling 'f' + + at /pwd/lang/eval-fail-scope-5.nix:6:7: + + 5| + 6| f = {x ? y, y ? x}: x + y; + | ^ + 7| + + error: infinite recursion encountered + + at /pwd/lang/eval-fail-scope-5.nix:6:12: + + 5| + 6| f = {x ? y, y ? x}: x + y; + | ^ + 7| diff --git a/tests/lang/eval-fail-seq.err.exp b/tests/lang/eval-fail-seq.err.exp new file mode 100644 index 000000000..33a7e9491 --- /dev/null +++ b/tests/lang/eval-fail-seq.err.exp @@ -0,0 +1,18 @@ +error: + … while calling the 'seq' builtin + + at /pwd/lang/eval-fail-seq.nix:1:1: + + 1| builtins.seq (abort "foo") 2 + | ^ + 2| + + … while calling the 'abort' builtin + + at /pwd/lang/eval-fail-seq.nix:1:15: + + 1| builtins.seq (abort "foo") 2 + | ^ + 2| + + error: evaluation aborted with the following error message: 'foo' diff --git a/tests/lang/eval-fail-set-override.err.exp b/tests/lang/eval-fail-set-override.err.exp new file mode 100644 index 000000000..beb29d678 --- /dev/null +++ b/tests/lang/eval-fail-set-override.err.exp @@ -0,0 +1,6 @@ +error: + … while evaluating the `__overrides` attribute + + at «none»:0: (source not available) + + error: value is an integer while a set was expected diff --git a/tests/lang/eval-fail-set-override.nix b/tests/lang/eval-fail-set-override.nix new file mode 100644 index 000000000..03551c186 --- /dev/null +++ b/tests/lang/eval-fail-set-override.nix @@ -0,0 +1 @@ +rec { __overrides = 1; } diff --git a/tests/lang/eval-fail-set.err.exp b/tests/lang/eval-fail-set.err.exp new file mode 100644 index 000000000..0d0140508 --- /dev/null +++ b/tests/lang/eval-fail-set.err.exp @@ -0,0 +1,7 @@ +error: undefined variable 'x' + + at /pwd/lang/eval-fail-set.nix:1:3: + + 1| 8.x + | ^ + 2| diff --git a/tests/lang/eval-fail-set.nix b/tests/lang/eval-fail-set.nix new file mode 100644 index 000000000..c6b7980b6 --- /dev/null +++ b/tests/lang/eval-fail-set.nix @@ -0,0 +1 @@ +8.x diff --git a/tests/lang/eval-fail-substring.err.exp b/tests/lang/eval-fail-substring.err.exp new file mode 100644 index 000000000..dc26a00bd --- /dev/null +++ b/tests/lang/eval-fail-substring.err.exp @@ -0,0 +1,12 @@ +error: + … while calling the 'substring' builtin + + at /pwd/lang/eval-fail-substring.nix:1:1: + + 1| builtins.substring (builtins.sub 0 1) 1 "x" + | ^ + 2| + + error: negative start position in 'substring' + + at «none»:0: (source not available) diff --git a/tests/lang/eval-fail-to-path.err.exp b/tests/lang/eval-fail-to-path.err.exp new file mode 100644 index 000000000..43ed2bdfc --- /dev/null +++ b/tests/lang/eval-fail-to-path.err.exp @@ -0,0 +1,14 @@ +error: + … while calling the 'toPath' builtin + + at /pwd/lang/eval-fail-to-path.nix:1:1: + + 1| builtins.toPath "foo/bar" + | ^ + 2| + + … while evaluating the first argument passed to builtins.toPath + + at «none»:0: (source not available) + + error: string 'foo/bar' doesn't represent an absolute path diff --git a/tests/lang/eval-fail-toJSON.err.exp b/tests/lang/eval-fail-toJSON.err.exp new file mode 100644 index 000000000..4e618c203 --- /dev/null +++ b/tests/lang/eval-fail-toJSON.err.exp @@ -0,0 +1,57 @@ +error: + … while calling the 'toJSON' builtin + + at /pwd/lang/eval-fail-toJSON.nix:1:1: + + 1| builtins.toJSON { + | ^ + 2| a.b = [ + + … while evaluating attribute 'a' + + at /pwd/lang/eval-fail-toJSON.nix:2:3: + + 1| builtins.toJSON { + 2| a.b = [ + | ^ + 3| true + + … while evaluating attribute 'b' + + at /pwd/lang/eval-fail-toJSON.nix:2:3: + + 1| builtins.toJSON { + 2| a.b = [ + | ^ + 3| true + + … while evaluating list element at index 3 + + … while evaluating attribute 'c' + + at /pwd/lang/eval-fail-toJSON.nix:7:7: + + 6| { + 7| c.d = throw "hah no"; + | ^ + 8| } + + … while evaluating attribute 'd' + + at /pwd/lang/eval-fail-toJSON.nix:7:7: + + 6| { + 7| c.d = throw "hah no"; + | ^ + 8| } + + … while calling the 'throw' builtin + + at /pwd/lang/eval-fail-toJSON.nix:7:13: + + 6| { + 7| c.d = throw "hah no"; + | ^ + 8| } + + error: hah no diff --git a/tests/lang/eval-fail-toJSON.nix b/tests/lang/eval-fail-toJSON.nix new file mode 100644 index 000000000..8112e1c1f --- /dev/null +++ b/tests/lang/eval-fail-toJSON.nix @@ -0,0 +1,10 @@ +builtins.toJSON { + a.b = [ + true + false + "it's a bird" + { + c.d = throw "hah no"; + } + ]; +} diff --git a/tests/lang/eval-fail-undeclared-arg.err.exp b/tests/lang/eval-fail-undeclared-arg.err.exp new file mode 100644 index 000000000..30db743c7 --- /dev/null +++ b/tests/lang/eval-fail-undeclared-arg.err.exp @@ -0,0 +1,17 @@ +error: + … from call site + + at /pwd/lang/eval-fail-undeclared-arg.nix:1:1: + + 1| ({x, z}: x + z) {x = "foo"; y = "bla"; z = "bar";} + | ^ + 2| + + error: function 'anonymous lambda' called with unexpected argument 'y' + + at /pwd/lang/eval-fail-undeclared-arg.nix:1:2: + + 1| ({x, z}: x + z) {x = "foo"; y = "bla"; z = "bar";} + | ^ + 2| + Did you mean one of x or z? diff --git a/tests/lang/eval-okay-flake-ref-to-string.exp b/tests/lang/eval-okay-flake-ref-to-string.exp new file mode 100644 index 000000000..110f8442d --- /dev/null +++ b/tests/lang/eval-okay-flake-ref-to-string.exp @@ -0,0 +1 @@ +"github:NixOS/nixpkgs/23.05?dir=lib" diff --git a/tests/lang/eval-okay-flake-ref-to-string.nix b/tests/lang/eval-okay-flake-ref-to-string.nix new file mode 100644 index 000000000..dbb4e5b2a --- /dev/null +++ b/tests/lang/eval-okay-flake-ref-to-string.nix @@ -0,0 +1,7 @@ +builtins.flakeRefToString { + type = "github"; + owner = "NixOS"; + repo = "nixpkgs"; + ref = "23.05"; + dir = "lib"; +} diff --git a/tests/lang/eval-okay-fromjson.nix b/tests/lang/eval-okay-fromjson.nix index e1c0f86cc..4c526b9ae 100644 --- a/tests/lang/eval-okay-fromjson.nix +++ b/tests/lang/eval-okay-fromjson.nix @@ -11,9 +11,12 @@ builtins.fromJSON "Width": 200, "Height": 250 }, + "Animated" : false, + "IDs": [116, 943, 234, 38793, true ,false,null, -100], + "Escapes": "\"\\\/\t\n\r\t", "Subtitle" : false, - "Latitude": 46.2051, - "Longitude": 6.0723 + "Latitude": 37.7668, + "Longitude": -122.3959 } } '' @@ -28,8 +31,11 @@ builtins.fromJSON Width = 200; Height = 250; }; + Animated = false; + IDs = [ 116 943 234 38793 true false null (0-100) ]; + Escapes = "\"\\\/\t\n\r\t"; # supported in JSON but not Nix: \b\f Subtitle = false; - Latitude = 46.2051; - Longitude = 6.0723; + Latitude = 37.7668; + Longitude = -122.3959; }; } diff --git a/tests/lang/eval-okay-merge-dynamic-attrs.exp b/tests/lang/eval-okay-merge-dynamic-attrs.exp new file mode 100644 index 000000000..157d677ce --- /dev/null +++ b/tests/lang/eval-okay-merge-dynamic-attrs.exp @@ -0,0 +1 @@ +{ set1 = { a = 1; b = 2; }; set2 = { a = 1; b = 2; }; set3 = { a = 1; b = 2; }; set4 = { a = 1; b = 2; }; } diff --git a/tests/lang/eval-okay-merge-dynamic-attrs.nix b/tests/lang/eval-okay-merge-dynamic-attrs.nix new file mode 100644 index 000000000..f459a554f --- /dev/null +++ b/tests/lang/eval-okay-merge-dynamic-attrs.nix @@ -0,0 +1,13 @@ +{ + set1 = { a = 1; }; + set1 = { "${"b" + ""}" = 2; }; + + set2 = { "${"b" + ""}" = 2; }; + set2 = { a = 1; }; + + set3.a = 1; + set3."${"b" + ""}" = 2; + + set4."${"b" + ""}" = 2; + set4.a = 1; +} diff --git a/tests/lang/eval-okay-overrides.nix b/tests/lang/eval-okay-overrides.nix index 358742b36..719bdc9c0 100644 --- a/tests/lang/eval-okay-overrides.nix +++ b/tests/lang/eval-okay-overrides.nix @@ -1,6 +1,6 @@ let - overrides = { a = 2; }; + overrides = { a = 2; b = 3; }; in (rec { __overrides = overrides; diff --git a/tests/lang/eval-okay-parse-flake-ref.exp b/tests/lang/eval-okay-parse-flake-ref.exp new file mode 100644 index 000000000..fc17ba085 --- /dev/null +++ b/tests/lang/eval-okay-parse-flake-ref.exp @@ -0,0 +1 @@ +{ dir = "lib"; owner = "NixOS"; ref = "23.05"; repo = "nixpkgs"; type = "github"; } diff --git a/tests/lang/eval-okay-parse-flake-ref.nix b/tests/lang/eval-okay-parse-flake-ref.nix new file mode 100644 index 000000000..db4ed2742 --- /dev/null +++ b/tests/lang/eval-okay-parse-flake-ref.nix @@ -0,0 +1 @@ + builtins.parseFlakeRef "github:NixOS/nixpkgs/23.05?dir=lib" diff --git a/tests/lang/eval-okay-print.err.exp b/tests/lang/eval-okay-print.err.exp new file mode 100644 index 000000000..3fc99be3e --- /dev/null +++ b/tests/lang/eval-okay-print.err.exp @@ -0,0 +1 @@ +trace: [ ] diff --git a/tests/lang/eval-okay-print.exp b/tests/lang/eval-okay-print.exp new file mode 100644 index 000000000..0d960fb70 --- /dev/null +++ b/tests/lang/eval-okay-print.exp @@ -0,0 +1 @@ +[ null [ [ «repeated» ] ] ] diff --git a/tests/lang/eval-okay-print.nix b/tests/lang/eval-okay-print.nix new file mode 100644 index 000000000..d36ba4da3 --- /dev/null +++ b/tests/lang/eval-okay-print.nix @@ -0,0 +1 @@ +with builtins; trace [(1+1)] [ null toString (deepSeq "x") (a: a) (let x=[x]; in x) ] diff --git a/tests/lang/eval-okay-search-path.flags b/tests/lang/eval-okay-search-path.flags index a28e68210..dfad1c611 100644 --- a/tests/lang/eval-okay-search-path.flags +++ b/tests/lang/eval-okay-search-path.flags @@ -1 +1 @@ --I lang/dir1 -I lang/dir2 -I dir5=lang/dir3 \ No newline at end of file +-I lang/dir1 -I lang/dir2 -I dir5=lang/dir3 diff --git a/tests/lang/framework.sh b/tests/lang/framework.sh new file mode 100644 index 000000000..516bff8ad --- /dev/null +++ b/tests/lang/framework.sh @@ -0,0 +1,33 @@ +# Golden test support +# +# Test that the output of the given test matches what is expected. If +# `_NIX_TEST_ACCEPT` is non-empty also update the expected output so +# that next time the test succeeds. +function diffAndAcceptInner() { + local -r testName=$1 + local -r got="$2" + local -r expected="$3" + + # Absence of expected file indicates empty output expected. + if test -e "$expected"; then + local -r expectedOrEmpty="$expected" + else + local -r expectedOrEmpty=lang/empty.exp + fi + + # Diff so we get a nice message + if ! diff --unified "$got" "$expectedOrEmpty"; then + echo "FAIL: evaluation result of $testName not as expected" + badDiff=1 + fi + + # Update expected if `_NIX_TEST_ACCEPT` is non-empty. + if test -n "${_NIX_TEST_ACCEPT-}"; then + cp "$got" "$expected" + # Delete empty expected files to avoid bloating the repo with + # empty files. + if ! test -s "$expected"; then + rm "$expected" + fi + fi +} diff --git a/tests/lang/parse-fail-dup-attrs-1.err.exp b/tests/lang/parse-fail-dup-attrs-1.err.exp new file mode 100644 index 000000000..4fe6b7a1f --- /dev/null +++ b/tests/lang/parse-fail-dup-attrs-1.err.exp @@ -0,0 +1,7 @@ +error: attribute 'x' already defined at «stdin»:1:3 + + at «stdin»:3:3: + + 2| y = 456; + 3| x = 789; + | ^ diff --git a/tests/lang/parse-fail-dup-attrs-2.err.exp b/tests/lang/parse-fail-dup-attrs-2.err.exp new file mode 100644 index 000000000..3aba2891f --- /dev/null +++ b/tests/lang/parse-fail-dup-attrs-2.err.exp @@ -0,0 +1,7 @@ +error: attribute 'x' already defined at «stdin»:9:5 + + at «stdin»:10:17: + + 9| x = 789; + 10| inherit (as) x; + | ^ diff --git a/tests/lang/parse-fail-dup-attrs-3.err.exp b/tests/lang/parse-fail-dup-attrs-3.err.exp new file mode 100644 index 000000000..3aba2891f --- /dev/null +++ b/tests/lang/parse-fail-dup-attrs-3.err.exp @@ -0,0 +1,7 @@ +error: attribute 'x' already defined at «stdin»:9:5 + + at «stdin»:10:17: + + 9| x = 789; + 10| inherit (as) x; + | ^ diff --git a/tests/lang/parse-fail-dup-attrs-4.err.exp b/tests/lang/parse-fail-dup-attrs-4.err.exp new file mode 100644 index 000000000..ff68446a1 --- /dev/null +++ b/tests/lang/parse-fail-dup-attrs-4.err.exp @@ -0,0 +1,7 @@ +error: attribute 'services.ssh.port' already defined at «stdin»:2:3 + + at «stdin»:3:3: + + 2| services.ssh.port = 22; + 3| services.ssh.port = 23; + | ^ diff --git a/tests/lang/parse-fail-dup-attrs-6.err.exp b/tests/lang/parse-fail-dup-attrs-6.err.exp new file mode 100644 index 000000000..74823fc25 --- /dev/null +++ b/tests/lang/parse-fail-dup-attrs-6.err.exp @@ -0,0 +1 @@ +error: attribute ‘services.ssh’ at (string):3:3 already defined at (string):2:3 diff --git a/tests/lang/parse-fail-dup-attrs-7.err.exp b/tests/lang/parse-fail-dup-attrs-7.err.exp new file mode 100644 index 000000000..512a499ca --- /dev/null +++ b/tests/lang/parse-fail-dup-attrs-7.err.exp @@ -0,0 +1,7 @@ +error: attribute 'x' already defined at «stdin»:6:12 + + at «stdin»:7:12: + + 6| inherit x; + 7| inherit x; + | ^ diff --git a/tests/lang/parse-fail-dup-formals.err.exp b/tests/lang/parse-fail-dup-formals.err.exp new file mode 100644 index 000000000..1d566fb33 --- /dev/null +++ b/tests/lang/parse-fail-dup-formals.err.exp @@ -0,0 +1,6 @@ +error: duplicate formal function argument 'x' + + at «stdin»:1:8: + + 1| {x, y, x}: x + | ^ diff --git a/tests/lang/parse-fail-eof-in-string.err.exp b/tests/lang/parse-fail-eof-in-string.err.exp new file mode 100644 index 000000000..f9fa72312 --- /dev/null +++ b/tests/lang/parse-fail-eof-in-string.err.exp @@ -0,0 +1,7 @@ +error: syntax error, unexpected end of file, expecting '"' + + at «stdin»:3:5: + + 2| # Note that this file must not end with a newline. + 3| a 1"$ + | ^ diff --git a/tests/lang/parse-fail-mixed-nested-attrs1.err.exp b/tests/lang/parse-fail-mixed-nested-attrs1.err.exp new file mode 100644 index 000000000..32f776795 --- /dev/null +++ b/tests/lang/parse-fail-mixed-nested-attrs1.err.exp @@ -0,0 +1,8 @@ +error: attribute 'z' already defined at «stdin»:3:16 + + at «stdin»:2:3: + + 1| { + 2| x.z = 3; + | ^ + 3| x = { y = 3; z = 3; }; diff --git a/tests/lang/parse-fail-mixed-nested-attrs2.err.exp b/tests/lang/parse-fail-mixed-nested-attrs2.err.exp new file mode 100644 index 000000000..0437cd50c --- /dev/null +++ b/tests/lang/parse-fail-mixed-nested-attrs2.err.exp @@ -0,0 +1,8 @@ +error: attribute 'y' already defined at «stdin»:3:9 + + at «stdin»:2:3: + + 1| { + 2| x.y.y = 3; + | ^ + 3| x = { y.y= 3; z = 3; }; diff --git a/tests/lang/parse-fail-patterns-1.err.exp b/tests/lang/parse-fail-patterns-1.err.exp new file mode 100644 index 000000000..634a04aaa --- /dev/null +++ b/tests/lang/parse-fail-patterns-1.err.exp @@ -0,0 +1,7 @@ +error: duplicate formal function argument 'args' + + at «stdin»:1:1: + + 1| args@{args, x, y, z}: x + | ^ + 2| diff --git a/tests/lang/parse-fail-regression-20060610.err.exp b/tests/lang/parse-fail-regression-20060610.err.exp new file mode 100644 index 000000000..167d01e85 --- /dev/null +++ b/tests/lang/parse-fail-regression-20060610.err.exp @@ -0,0 +1,8 @@ +error: undefined variable 'gcc' + + at «stdin»:8:12: + + 7| + 8| body = ({ + | ^ + 9| inherit gcc; diff --git a/tests/lang/parse-fail-undef-var-2.err.exp b/tests/lang/parse-fail-undef-var-2.err.exp new file mode 100644 index 000000000..77c96bbd2 --- /dev/null +++ b/tests/lang/parse-fail-undef-var-2.err.exp @@ -0,0 +1,7 @@ +error: syntax error, unexpected ':', expecting '}' + + at «stdin»:3:13: + + 2| + 3| f = {x, y : + | ^ diff --git a/tests/lang/parse-fail-undef-var.err.exp b/tests/lang/parse-fail-undef-var.err.exp new file mode 100644 index 000000000..48e88747f --- /dev/null +++ b/tests/lang/parse-fail-undef-var.err.exp @@ -0,0 +1,7 @@ +error: undefined variable 'y' + + at «stdin»:1:4: + + 1| x: y + | ^ + 2| diff --git a/tests/lang/parse-fail-utf8.err.exp b/tests/lang/parse-fail-utf8.err.exp new file mode 100644 index 000000000..6087479a3 --- /dev/null +++ b/tests/lang/parse-fail-utf8.err.exp @@ -0,0 +1,6 @@ +error: syntax error, unexpected invalid token, expecting end of file + + at «stdin»:1:5: + + 1| 123 + | ^ diff --git a/tests/lang/parse-fail-uft8.nix b/tests/lang/parse-fail-utf8.nix similarity index 100% rename from tests/lang/parse-fail-uft8.nix rename to tests/lang/parse-fail-utf8.nix diff --git a/tests/lang/parse-okay-1.exp b/tests/lang/parse-okay-1.exp new file mode 100644 index 000000000..d5ab5f18a --- /dev/null +++ b/tests/lang/parse-okay-1.exp @@ -0,0 +1 @@ +({ x, y, z }: ((x + y) + z)) diff --git a/tests/lang/parse-okay-crlf.exp b/tests/lang/parse-okay-crlf.exp new file mode 100644 index 000000000..4213609fc --- /dev/null +++ b/tests/lang/parse-okay-crlf.exp @@ -0,0 +1 @@ +rec { foo = "multi\nline\n string\n test\r"; x = y; y = 123; z = 456; } diff --git a/tests/lang/parse-okay-dup-attrs-5.exp b/tests/lang/parse-okay-dup-attrs-5.exp new file mode 100644 index 000000000..88b0b036f --- /dev/null +++ b/tests/lang/parse-okay-dup-attrs-5.exp @@ -0,0 +1 @@ +{ services = { ssh = { enable = true; port = 23; }; }; } diff --git a/tests/lang/parse-okay-dup-attrs-6.exp b/tests/lang/parse-okay-dup-attrs-6.exp new file mode 100644 index 000000000..88b0b036f --- /dev/null +++ b/tests/lang/parse-okay-dup-attrs-6.exp @@ -0,0 +1 @@ +{ services = { ssh = { enable = true; port = 23; }; }; } diff --git a/tests/lang/parse-okay-mixed-nested-attrs-1.exp b/tests/lang/parse-okay-mixed-nested-attrs-1.exp new file mode 100644 index 000000000..89c66f760 --- /dev/null +++ b/tests/lang/parse-okay-mixed-nested-attrs-1.exp @@ -0,0 +1 @@ +{ x = { q = 3; y = 3; z = 3; }; } diff --git a/tests/lang/parse-okay-mixed-nested-attrs-2.exp b/tests/lang/parse-okay-mixed-nested-attrs-2.exp new file mode 100644 index 000000000..89c66f760 --- /dev/null +++ b/tests/lang/parse-okay-mixed-nested-attrs-2.exp @@ -0,0 +1 @@ +{ x = { q = 3; y = 3; z = 3; }; } diff --git a/tests/lang/parse-okay-mixed-nested-attrs-3.exp b/tests/lang/parse-okay-mixed-nested-attrs-3.exp new file mode 100644 index 000000000..b89a59734 --- /dev/null +++ b/tests/lang/parse-okay-mixed-nested-attrs-3.exp @@ -0,0 +1 @@ +{ services = { httpd = { enable = true; }; ssh = { enable = true; port = 123; }; }; } diff --git a/tests/lang/parse-okay-regression-20041027.exp b/tests/lang/parse-okay-regression-20041027.exp new file mode 100644 index 000000000..9df7219e4 --- /dev/null +++ b/tests/lang/parse-okay-regression-20041027.exp @@ -0,0 +1 @@ +({ fetchurl, stdenv }: ((stdenv).mkDerivation { name = "libXi-6.0.1"; src = (fetchurl { md5 = "7e935a42428d63a387b3c048be0f2756"; url = "http://freedesktop.org/~xlibs/release/libXi-6.0.1.tar.bz2"; }); })) diff --git a/tests/lang/parse-okay-regression-751.exp b/tests/lang/parse-okay-regression-751.exp new file mode 100644 index 000000000..e2ed886fe --- /dev/null +++ b/tests/lang/parse-okay-regression-751.exp @@ -0,0 +1 @@ +(let const = (a: "const"); in ((const { x = "q"; }))) diff --git a/tests/lang/parse-okay-subversion.exp b/tests/lang/parse-okay-subversion.exp new file mode 100644 index 000000000..4168ee8bf --- /dev/null +++ b/tests/lang/parse-okay-subversion.exp @@ -0,0 +1 @@ +({ fetchurl, localServer ? false, httpServer ? false, sslSupport ? false, pythonBindings ? false, javaSwigBindings ? false, javahlBindings ? false, stdenv, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null }: assert (expat != null); assert (localServer -> (db4 != null)); assert (httpServer -> ((httpd != null) && ((httpd).expat == expat))); assert (sslSupport -> ((openssl != null) && (httpServer -> ((httpd).openssl == openssl)))); assert (pythonBindings -> ((swig != null) && (swig).pythonSupport)); assert (javaSwigBindings -> ((swig != null) && (swig).javaSupport)); assert (javahlBindings -> (j2sdk != null)); ((stdenv).mkDerivation { builder = /foo/bar; db4 = (if localServer then db4 else null); inherit expat ; inherit httpServer ; httpd = (if httpServer then httpd else null); j2sdk = (if javaSwigBindings then (swig).j2sdk else (if javahlBindings then j2sdk else null)); inherit javaSwigBindings ; inherit javahlBindings ; inherit localServer ; name = "subversion-1.1.1"; openssl = (if sslSupport then openssl else null); patches = (if javahlBindings then [ (/javahl.patch) ] else [ ]); python = (if pythonBindings then (swig).python else null); inherit pythonBindings ; src = (fetchurl { md5 = "a180c3fe91680389c210c99def54d9e0"; url = "http://subversion.tigris.org/tarballs/subversion-1.1.1.tar.bz2"; }); inherit sslSupport ; swig = (if (pythonBindings || javaSwigBindings) then swig else null); })) diff --git a/tests/lang/parse-okay-url.exp b/tests/lang/parse-okay-url.exp new file mode 100644 index 000000000..e5f0829b0 --- /dev/null +++ b/tests/lang/parse-okay-url.exp @@ -0,0 +1 @@ +[ ("x:x") ("https://svn.cs.uu.nl:12443/repos/trace/trunk") ("http://www2.mplayerhq.hu/MPlayer/releases/fonts/font-arial-iso-8859-1.tar.bz2") ("http://losser.st-lab.cs.uu.nl/~armijn/.nix/gcc-3.3.4-static-nix.tar.gz") ("http://fpdownload.macromedia.com/get/shockwave/flash/english/linux/7.0r25/install_flash_player_7_linux.tar.gz") ("https://ftp5.gwdg.de/pub/linux/archlinux/extra/os/x86_64/unzip-6.0-14-x86_64.pkg.tar.zst") ("ftp://ftp.gtk.org/pub/gtk/v1.2/gtk+-1.2.10.tar.gz") ] diff --git a/tests/local.mk b/tests/local.mk index ca4e47cf9..4edf31303 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -14,12 +14,12 @@ nix_tests = \ flakes/absolute-paths.sh \ flakes/build-paths.sh \ flakes/flake-in-submodule.sh \ - ca/gc.sh \ gc.sh \ nix-collect-garbage-d.sh \ remote-store.sh \ legacy-ssh-store.sh \ lang.sh \ + lang-test-infra.sh \ experimental-features.sh \ fetchMercurial.sh \ gc-auto.sh \ @@ -27,8 +27,6 @@ nix_tests = \ user-envs-migration.sh \ binary-cache.sh \ multiple-outputs.sh \ - ca/build.sh \ - ca/new-build-cmd.sh \ nix-build.sh \ gc-concurrent.sh \ repair.sh \ @@ -46,24 +44,17 @@ nix_tests = \ referrers.sh \ optimise-store.sh \ substitute-with-invalid-ca.sh \ - ca/concurrent-builds.sh \ signing.sh \ - ca/build-with-garbage-path.sh \ hash.sh \ gc-non-blocking.sh \ check.sh \ - ca/substitute.sh \ nix-shell.sh \ - ca/signatures.sh \ - ca/nix-shell.sh \ - ca/nix-copy.sh \ check-refs.sh \ build-remote-input-addressed.sh \ secure-drv-outputs.sh \ restricted.sh \ fetchGitSubmodules.sh \ flakes/search-root.sh \ - ca/duplicate-realisation-in-closure.sh \ readfile-context.sh \ nix-channel.sh \ recursive.sh \ @@ -79,9 +70,7 @@ nix_tests = \ nar-access.sh \ pure-eval.sh \ eval.sh \ - ca/post-hook.sh \ repl.sh \ - ca/recursive.sh \ binary-cache-build-remote.sh \ search.sh \ logging.sh \ @@ -93,6 +82,7 @@ nix_tests = \ misc.sh \ dump-db.sh \ linux-sandbox.sh \ + supplementary-groups.sh \ build-dry.sh \ structured-attrs.sh \ shell.sh \ @@ -100,19 +90,15 @@ nix_tests = \ zstd.sh \ compression-levels.sh \ nix-copy-ssh.sh \ + nix-copy-ssh-ng.sh \ post-hook.sh \ function-trace.sh \ flakes/config.sh \ fmt.sh \ eval-store.sh \ why-depends.sh \ - ca/why-depends.sh \ derivation-json.sh \ - ca/derivation-json.sh \ import-derivation.sh \ - ca/import-derivation.sh \ - dyn-drv/text-hashed-output.sh \ - dyn-drv/recursive-mod-json.sh \ nix_path.sh \ case-hack.sh \ placeholders.sh \ @@ -121,8 +107,7 @@ nix_tests = \ build.sh \ build-delete.sh \ output-normalization.sh \ - ca/nix-run.sh \ - selfref-gc.sh ca/selfref-gc.sh \ + selfref-gc.sh \ db-migration.sh \ bash-profile.sh \ pass-as-file.sh \ @@ -136,7 +121,8 @@ nix_tests = \ path-from-hash-part.sh \ test-libstoreconsumer.sh \ toString-path.sh \ - read-only-store.sh + read-only-store.sh \ + nested-sandboxing.sh ifeq ($(HAVE_LIBCPUID), 1) nix_tests += compute-levels.sh @@ -146,16 +132,12 @@ install-tests += $(foreach x, $(nix_tests), $(d)/$(x)) clean-files += \ $(d)/common/vars-and-functions.sh \ - $(d)/config.nix \ - $(d)/ca/config.nix \ - $(d)/dyn-drv/config.nix + $(d)/config.nix test-deps += \ tests/common/vars-and-functions.sh \ tests/config.nix \ - tests/ca/config.nix \ - tests/test-libstoreconsumer/test-libstoreconsumer \ - tests/dyn-drv/config.nix + tests/test-libstoreconsumer/test-libstoreconsumer ifeq ($(BUILD_SHARED_LIBS), 1) test-deps += tests/plugins/libplugintest.$(SO_EXT) diff --git a/tests/misc.sh b/tests/misc.sh index 60d58310e..af96d20bd 100644 --- a/tests/misc.sh +++ b/tests/misc.sh @@ -24,3 +24,9 @@ eval_stdin_res=$(echo 'let a = {} // a; in a.foo' | nix-instantiate --eval -E - echo $eval_stdin_res | grep "at «stdin»:1:15:" echo $eval_stdin_res | grep "infinite recursion encountered" +# Attribute path errors +expectStderr 1 nix-instantiate --eval -E '{}' -A '"x' | grepQuiet "missing closing quote in selection path" +expectStderr 1 nix-instantiate --eval -E '[]' -A 'x' | grepQuiet "should be a set" +expectStderr 1 nix-instantiate --eval -E '{}' -A '1' | grepQuiet "should be a list" +expectStderr 1 nix-instantiate --eval -E '{}' -A '.' | grepQuiet "empty attribute name" +expectStderr 1 nix-instantiate --eval -E '[]' -A '1' | grepQuiet "out of range" diff --git a/tests/nested-sandboxing.sh b/tests/nested-sandboxing.sh new file mode 100644 index 000000000..d9fa788aa --- /dev/null +++ b/tests/nested-sandboxing.sh @@ -0,0 +1,11 @@ +source common.sh +# This test is run by `tests/nested-sandboxing/runner.nix` in an extra layer of sandboxing. +[[ -d /nix/store ]] || skipTest "running this test without Nix's deps being drawn from /nix/store is not yet supported" + +requireSandboxSupport + +source ./nested-sandboxing/command.sh + +expectStderr 100 runNixBuild badStoreUrl 2 | grepQuiet '`sandbox-build-dir` must not contain' + +runNixBuild goodStoreUrl 5 diff --git a/tests/nested-sandboxing/command.sh b/tests/nested-sandboxing/command.sh new file mode 100644 index 000000000..69366486c --- /dev/null +++ b/tests/nested-sandboxing/command.sh @@ -0,0 +1,29 @@ +export NIX_BIN_DIR=$(dirname $(type -p nix)) +# TODO Get Nix and its closure more flexibly +export EXTRA_SANDBOX="/nix/store $(dirname $NIX_BIN_DIR)" + +badStoreUrl () { + local altitude=$1 + echo $TEST_ROOT/store-$altitude +} + +goodStoreUrl () { + local altitude=$1 + echo $("badStoreUrl" "$altitude")?store=/foo-$altitude +} + +# The non-standard sandbox-build-dir helps ensure that we get the same behavior +# whether this test is being run in a derivation as part of the nix build or +# being manually run by a developer outside a derivation +runNixBuild () { + local storeFun=$1 + local altitude=$2 + nix-build \ + --no-substitute --no-out-link \ + --store "$("$storeFun" "$altitude")" \ + --extra-sandbox-paths "$EXTRA_SANDBOX" \ + ./nested-sandboxing/runner.nix \ + --arg altitude "$((altitude - 1))" \ + --argstr storeFun "$storeFun" \ + --sandbox-build-dir /build-non-standard +} diff --git a/tests/nested-sandboxing/runner.nix b/tests/nested-sandboxing/runner.nix new file mode 100644 index 000000000..9a5822c88 --- /dev/null +++ b/tests/nested-sandboxing/runner.nix @@ -0,0 +1,24 @@ +{ altitude, storeFun }: + +with import ../config.nix; + +mkDerivation { + name = "nested-sandboxing"; + busybox = builtins.getEnv "busybox"; + EXTRA_SANDBOX = builtins.getEnv "EXTRA_SANDBOX"; + buildCommand = if altitude == 0 then '' + echo Deep enough! > $out + '' else '' + cp -r ${../common} ./common + cp ${../common.sh} ./common.sh + cp ${../config.nix} ./config.nix + cp -r ${./.} ./nested-sandboxing + + export PATH=${builtins.getEnv "NIX_BIN_DIR"}:$PATH + + source common.sh + source ./nested-sandboxing/command.sh + + runNixBuild ${storeFun} ${toString altitude} >> $out + ''; +} diff --git a/tests/nix-copy-ssh-ng.sh b/tests/nix-copy-ssh-ng.sh new file mode 100644 index 000000000..45e53c9c0 --- /dev/null +++ b/tests/nix-copy-ssh-ng.sh @@ -0,0 +1,18 @@ +source common.sh + +clearStore +clearCache + +remoteRoot=$TEST_ROOT/store2 +chmod -R u+w "$remoteRoot" || true +rm -rf "$remoteRoot" + +outPath=$(nix-build --no-out-link dependencies.nix) + +nix store ping --store "ssh-ng://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" + +# Regression test for https://github.com/NixOS/nix/issues/6253 +nix copy --to "ssh-ng://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath --no-check-sigs & +nix copy --to "ssh-ng://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath --no-check-sigs + +[ -f $remoteRoot$outPath/foobar ] diff --git a/tests/nixos/nix-copy.nix b/tests/nixos/nix-copy.nix index 16c477bf9..ef053de03 100644 --- a/tests/nixos/nix-copy.nix +++ b/tests/nixos/nix-copy.nix @@ -79,6 +79,15 @@ in { server.copy_from_host("key.pub", "/root/.ssh/authorized_keys") server.succeed("systemctl restart sshd") client.succeed(f"ssh -o StrictHostKeyChecking=no {server.name} 'echo hello world'") + client.succeed(f"ssh -O check {server.name}") + client.succeed(f"ssh -O exit {server.name}") + client.fail(f"ssh -O check {server.name}") + + # Check that an explicit master will work + client.succeed(f"ssh -MNfS /tmp/master {server.name}") + client.succeed(f"ssh -S /tmp/master -O check {server.name}") + client.succeed("NIX_SSHOPTS='-oControlPath=/tmp/master' nix copy --to ssh://server ${pkgA} >&2") + client.succeed(f"ssh -S /tmp/master -O exit {server.name}") # Copy the closure of package B from the server to the client, using ssh-ng. client.fail("nix-store --check-validity ${pkgB}") diff --git a/tests/restricted.sh b/tests/restricted.sh index 776893a56..17f310a4b 100644 --- a/tests/restricted.sh +++ b/tests/restricted.sh @@ -49,3 +49,5 @@ output="$(nix eval --raw --restrict-eval -I "$traverseDir" \ 2>&1 || :)" echo "$output" | grep "is forbidden" echo "$output" | grepInverse -F restricted-secret + +expectStderr 1 nix-instantiate --restrict-eval true ./dependencies.nix | grepQuiet "forbidden in restricted mode" diff --git a/tests/supplementary-groups.sh b/tests/supplementary-groups.sh new file mode 100644 index 000000000..d18fb2414 --- /dev/null +++ b/tests/supplementary-groups.sh @@ -0,0 +1,37 @@ +source common.sh + +requireSandboxSupport +[[ $busybox =~ busybox ]] || skipTest "no busybox" +if ! command -p -v unshare; then skipTest "Need unshare"; fi +needLocalStore "The test uses --store always so we would just be bypassing the daemon" + +unshare --mount --map-root-user bash < paths { DerivedPath::Built { - .drvPath = store->parseStorePath(drvPath), + .drvPath = makeConstantStorePathRef(store->parseStorePath(drvPath)), .outputs = OutputsSpec::Names{"out"} } }; diff --git a/tests/why-depends.sh b/tests/why-depends.sh index b35a0d1cf..9680bf80e 100644 --- a/tests/why-depends.sh +++ b/tests/why-depends.sh @@ -22,3 +22,8 @@ echo "$PRECISE_WHY_DEPENDS_OUTPUT" | grepQuiet input-2 # But only the “precise” one should refer to `reference-to-input-2` echo "$FAST_WHY_DEPENDS_OUTPUT" | grepQuietInverse reference-to-input-2 echo "$PRECISE_WHY_DEPENDS_OUTPUT" | grepQuiet reference-to-input-2 + +<<<"$PRECISE_WHY_DEPENDS_OUTPUT" sed -n '2p' | grepQuiet "└───reference-to-input-2 -> " +<<<"$PRECISE_WHY_DEPENDS_OUTPUT" sed -n '3p' | grep " →" | grepQuiet "dependencies-input-2" +<<<"$PRECISE_WHY_DEPENDS_OUTPUT" sed -n '4p' | grepQuiet " └───input0: …" # in input-2, file input0 +<<<"$PRECISE_WHY_DEPENDS_OUTPUT" sed -n '5p' | grep " →" | grepQuiet "dependencies-input-0" # is dependencies-input-0 referenced