max/nix-super
1
0
Fork 0
mirror of https://github.com/privatevoid-net/nix-super.git synced 2024-11-15 02:36:16 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'nixos/master'

Browse source
This commit is contained in:
Max Headroom 2023-08-13 20:15:13 +02:00
commit 5d7ee4d437
228 changed files with 4151 additions and 1425 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/labels.yml vendored
View file

@ -21,4 +21,4 @@ jobs:
- uses: actions/labeler@v4
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
sync-labels: true
sync-labels: false

1
.gitignore vendored
View file

@ -95,6 +95,7 @@ perl/Makefile.config
# /tests/lang/
/tests/lang/*.out
/tests/lang/*.out.xml
/tests/lang/*.err
/tests/lang/*.ast
/perl/lib/Nix/Config.pm

2
.version
View file

@ -1 +1 @@
2.17.0
2.18.0

2
Makefile
View file

@ -27,6 +27,8 @@ makefiles += \
src/libstore/tests/local.mk \
src/libexpr/tests/local.mk \
tests/local.mk \
tests/ca/local.mk \
tests/dyn-drv/local.mk \
tests/test-libstoreconsumer/local.mk \
tests/plugins/local.mk
else

9
configure.ac
View file

@ -5,7 +5,14 @@ AC_CONFIG_AUX_DIR(config)
AC_PROG_SED
# Construct a Nix system name (like "i686-linux").
# Construct a Nix system name (like "i686-linux"):
# https://www.gnu.org/software/autoconf/manual/html_node/Canonicalizing.html#index-AC_005fCANONICAL_005fHOST-1
# The inital value is produced by the `config/config.guess` script:
# upstream: https://git.savannah.gnu.org/cgit/config.git/tree/config.guess
# It has the following form, which is not documented anywhere:
# <cpu>-<vendor>-<os>[<version>][-<abi>]
# If `./configure` is passed any of the `--host`, `--build`, `--target` options, the value comes from `config/config.sub` instead:
# upstream: https://git.savannah.gnu.org/cgit/config.git/tree/config.sub
AC_CANONICAL_HOST
AC_MSG_CHECKING([for the canonical Nix system name])

8
doc/manual/generate-builtin-constants.nix
View file

@ -10,12 +10,14 @@ let
type' = optionalString (type != null) " (${type})";
impureNotice = optionalString impure-only ''
Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval).
> **Note**
>
> Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval).
'';
in
squash ''
<dt id="builtin-constants-${name}">
<a href="#builtin-constants-${name}"><code>${name}</code>${type'}</a>
<dt id="builtins-${name}">
<a href="#builtins-${name}"><code>${name}</code></a>${type'}
</dt>
<dd>

21
doc/manual/generate-manpage.nix
View file

@ -137,12 +137,29 @@ let
storeDocs =
let
showStore = name: { settings, doc }:
''
showStore = name: { settings, doc, experimentalFeature }:
let
experimentalFeatureNote = optionalString (experimentalFeature != null) ''
> **Warning**
> This store is part of an
> [experimental feature](@docroot@/contributing/experimental-features.md).
To use this store, you need to make sure the corresponding experimental feature,
[`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}),
is enabled.
For example, include the following in [`nix.conf`](#):
```
extra-experimental-features = ${experimentalFeature}
```
'';
in ''
## ${name}
${doc}
${experimentalFeatureNote}
**Settings**:
${showSettings { useAnchors = false; } settings}

2
doc/manual/redirects.js
View file

@ -281,7 +281,7 @@ const redirects = {
"chap-introduction": "introduction.html",
"ch-basic-package-mgmt": "package-management/basic-package-mgmt.html",
"ssec-binary-cache-substituter": "package-management/binary-cache-substituter.html",
"sec-channels": "package-management/channels.html",
"sec-channels": "command-ref/nix-channel.html",
"ssec-copy-closure": "package-management/copy-closure.html",
"sec-garbage-collection": "package-management/garbage-collection.html",
"ssec-gc-roots": "package-management/garbage-collector-roots.html",

2
doc/manual/src/SUMMARY.md.in
View file

@ -21,7 +21,6 @@
- [Profiles](package-management/profiles.md)
- [Garbage Collection](package-management/garbage-collection.md)
- [Garbage Collector Roots](package-management/garbage-collector-roots.md)
- [Channels](package-management/channels.md)
- [Sharing Packages Between Machines](package-management/sharing-packages.md)
- [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md)
- [Copying Closures via SSH](package-management/copy-closure.md)
@ -110,6 +109,7 @@
- [C++ style guide](contributing/cxx.md)
- [Release Notes](release-notes/release-notes.md)
- [Release X.Y (202?-??-??)](release-notes/rl-next.md)
- [Release 2.17 (2023-07-24)](release-notes/rl-2.17.md)
- [Release 2.16 (2023-05-31)](release-notes/rl-2.16.md)
- [Release 2.15 (2023-04-11)](release-notes/rl-2.15.md)
- [Release 2.14 (2023-02-28)](release-notes/rl-2.14.md)

75
doc/manual/src/command-ref/nix-channel.md
View file

@ -8,36 +8,46 @@
# Description
A Nix channel is a mechanism that allows you to automatically stay
up-to-date with a set of pre-built Nix expressions. A Nix channel is
just a URL that points to a place containing a set of Nix expressions.
Channels are a mechanism for referencing remote Nix expressions and conveniently retrieving their latest version.
To see the list of official NixOS channels, visit
<https://nixos.org/channels>.
The moving parts of channels are:
- The official channels listed at <https://nixos.org/channels>
- The user-specific list of [subscribed channels](#subscribed-channels)
- The [downloaded channel contents](#channels)
- The [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path), set with the [`-I` option](#opt-i) or the [`NIX_PATH` environment variable](#env-NIX_PATH)
> **Note**
>
> The state of a subscribed channel is external to the Nix expressions relying on it.
> This may limit reproducibility.
>
> Dependencies on other Nix expressions can be declared explicitly with:
> - [`fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl), [`fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball), or [`fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) in Nix expressions
> - the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I) in command line invocations
This command has the following operations:
- `--add` *url* \[*name*\]\
Adds a channel named *name* with URL *url* to the list of subscribed
channels. If *name* is omitted, it defaults to the last component of
*url*, with the suffixes `-stable` or `-unstable` removed.
Add a channel *name* located at *url* to the list of subscribed channels.
If *name* is omitted, default to the last component of *url*, with the suffixes `-stable` or `-unstable` removed.
> **Note**
>
> `--add` does not automatically perform an update.
> Use `--update` explicitly.
A channel URL must point to a directory containing a file `nixexprs.tar.gz`.
At the top level, that tarball must contain a single directory with a `default.nix` file that serves as the channel’s entry point.
- `--remove` *name*\
Removes the channel named *name* from the list of subscribed
channels.
Remove the channel *name* from the list of subscribed channels.
- `--list`\
Prints the names and URLs of all subscribed channels on standard
output.
Print the names and URLs of all subscribed channels on standard output.
- `--update` \[*names*…\]\
Downloads the Nix expressions of all subscribed channels (or only
those included in *names* if specified) and makes them the default
for `nix-env` operations (by symlinking them from the directory
`~/.nix-defexpr`).
Download the Nix expressions of subscribed channels and create a new generation.
Update all channels if none is specified, and only those included in *names* otherwise.
- `--list-generations`\
Prints a list of all the current existing generations for the
@ -49,13 +59,8 @@ This command has the following operations:
```
- `--rollback` \[*generation*\]\
Reverts the previous call to `nix-channel
--update`. Optionally, you can specify a specific channel generation
number to restore.
Note that `--add` does not automatically perform an update.
The list of subscribed channels is stored in `~/.nix-channels`.
Revert channels to the state before the last call to `nix-channel --update`.
Optionally, you can specify a specific channel *generation* number to restore.
{{#include ./opt-common.md}}
@ -69,23 +74,33 @@ The list of subscribed channels is stored in `~/.nix-channels`.
# Examples
To subscribe to the Nixpkgs channel and install the GNU Hello package:
Subscribe to the Nixpkgs channel and run `hello` from the GNU Hello package:
```console
$ nix-channel --add https://nixos.org/channels/nixpkgs-unstable
$ nix-channel --list
nixpkgs https://nixos.org/channels/nixpkgs
$ nix-channel --update
$ nix-env --install --attr nixpkgs.hello
$ nix-shell -p hello --run hello
hello
```
You can revert channel updates using `--rollback`:
Revert channel updates using `--rollback`:
```console
$ nix-instantiate --eval --expr '(import <nixpkgs> {}).lib.version'
"14.04.527.0e935f1"
$ nix-instantiate --eval '<nixpkgs>' --attr lib.version
"22.11pre296212.530a53dcbc9"
$ nix-channel --rollback
switching from generation 483 to 482
$ nix-instantiate --eval --expr '(import <nixpkgs> {}).lib.version'
"14.04.526.dbadfad"
$ nix-instantiate --eval '<nixpkgs>' --attr lib.version
"22.11pre281526.d0419badfad"
```
Remove a channel:
```console
$ nix-channel --remove nixpkgs
$ nix-channel --list
```

71
doc/manual/src/contributing/hacking.md
View file

@ -110,41 +110,72 @@ You can also build Nix for one of the [supported platforms](#platforms).
## Platforms
As specified in [`flake.nix`], Nix can be built for various platforms:
- `aarch64-linux`
- `i686-linux`
- `x86_64-darwin`
- `x86_64-linux`
Nix can be built for various platforms, as specified in [`flake.nix`]:
[`flake.nix`]: https://github.com/nixos/nix/blob/master/flake.nix
- `x86_64-linux`
- `x86_64-darwin`
- `i686-linux`
- `aarch64-linux`
- `aarch64-darwin`
- `armv6l-linux`
- `armv7l-linux`
In order to build Nix for a different platform than the one you're currently
on, you need to have some way for your system Nix to build code for that
platform. Common solutions include [remote builders] and [binfmt emulation]
on, you need a way for your current Nix installation to build code for that
platform. Common solutions include [remote builders] and [binary format emulation]
(only supported on NixOS).
[remote builders]: ../advanced-topics/distributed-builds.md
[binfmt emulation]: https://nixos.org/manual/nixos/stable/options.html#opt-boot.binfmt.emulatedSystems
[binary format emulation]: https://nixos.org/manual/nixos/stable/options.html#opt-boot.binfmt.emulatedSystems
These solutions let Nix perform builds as if you're on the native platform, so
executing the build is as simple as
```console
$ nix build .#packages.aarch64-linux.default
```
for flake-enabled Nix, or
Given such a setup, executing the build only requires selecting the respective attribute.
For example, to compile for `aarch64-linux`:
```console
$ nix-build --attr packages.aarch64-linux.default
```
for classic Nix.
or for Nix with the [`flakes`] and [`nix-command`] experimental features enabled:
You can use any of the other supported platforms in place of `aarch64-linux`.
```console
$ nix build .#packages.aarch64-linux.default
```
Cross-compiled builds are available for ARMv6 and ARMv7, and Nix on unsupported platforms can be bootstrapped by adding more `crossSystems` in `flake.nix`.
Cross-compiled builds are available for ARMv6 (`armv6l-linux`) and ARMv7 (`armv7l-linux`).
Add more [system types](#system-type) to `crossSystems` in `flake.nix` to bootstrap Nix on unsupported platforms.
## System type
Nix uses a string with he following format to identify the *system type* or *platform* it runs on:
```
<cpu>-<os>[-<abi>]
```
It is set when Nix is compiled for the given system, and based on the output of [`config.guess`](https://github.com/nixos/nix/blob/master/config/config.guess) ([upstream](https://git.savannah.gnu.org/cgit/config.git/tree/config.guess)):
```
<cpu>-<vendor>-<os>[<version>][-<abi>]
```
When Nix is built such that `./configure` is passed any of the `--host`, `--build`, `--target` options, the value is based on the output of [`config.sub`](https://github.com/nixos/nix/blob/master/config/config.sub) ([upstream](https://git.savannah.gnu.org/cgit/config.git/tree/config.sub)):
```
<cpu>-<vendor>[-<kernel>]-<os>
```
For historic reasons and backward-compatibility, some CPU and OS identifiers are translated from the GNU Autotools naming convention in [`configure.ac`](https://github.com/nixos/nix/blob/master/configure.ac) as follows:
| `config.guess` | Nix |
|----------------------------|---------------------|
| `amd64` | `x86_64` |
| `i*86` | `i686` |
| `arm6` | `arm6l` |
| `arm7` | `arm7l` |
| `linux-gnu*` | `linux` |
| `linux-musl*` | `linux` |
## Compilation environments

54
doc/manual/src/contributing/testing.md
View file

@ -14,6 +14,8 @@ You can run the whole testsuite with `make check`, or the tests for a specific c
The functional tests reside under the `tests` directory and are listed in `tests/local.mk`.
Each test is a bash script.
### Running the whole test suite
The whole test suite can be run with:
```shell-session
@ -23,6 +25,33 @@ ran test tests/bar.sh... [PASS]
...
```
### Grouping tests
Sometimes it is useful to group related tests so they can be easily run together without running the entire test suite.
Each test group is in a subdirectory of `tests`.
For example, `tests/ca/local.mk` defines a `ca` test group for content-addressed derivation outputs.
That test group can be run like this:
```shell-session
$ make ca.test-group -j50
ran test tests/ca/nix-run.sh... [PASS]
ran test tests/ca/import-derivation.sh... [PASS]
...
```
The test group is defined in Make like this:
```makefile
$(test-group-name)-tests := \
$(d)/test0.sh \
$(d)/test1.sh \
...
install-tests-groups += $(test-group-name)
```
### Running individual tests
Individual tests can be run with `make`:
```shell-session
@ -86,6 +115,31 @@ GNU gdb (GDB) 12.1
One can debug the Nix invocation in all the usual ways.
For example, enter `run` to start the Nix invocation.
### Characterization testing
Occasionally, Nix utilizes a technique called [Characterization Testing](https://en.wikipedia.org/wiki/Characterization_test) as part of the functional tests.
This technique is to include the exact output/behavior of a former version of Nix in a test in order to check that Nix continues to produce the same behavior going forward.
For example, this technique is used for the language tests, to check both the printed final value if evaluation was successful, and any errors and warnings encountered.
It is frequently useful to regenerate the expected output.
To do that, rerun the failed test with `_NIX_TEST_ACCEPT=1`.
(At least, this is the convention we've used for `tests/lang.sh`.
If we add more characterization testing we should always strive to be consistent.)
An interesting situation to document is the case when these tests are "overfitted".
The language tests are, again, an example of this.
The expected successful output of evaluation is supposed to be highly stable – we do not intend to make breaking changes to (the stable parts of) the Nix language.
However, the errors and warnings during evaluation (successful or not) are not stable in this way.
We are free to change how they are displayed at any time.
It may be surprising that we would test non-normative behavior like diagnostic outputs.
Diagnostic outputs are indeed not a stable interface, but they still are important to users.
By recording the expected output, the test suite guards against accidental changes, and ensure the *result* (not just the code that implements it) of the diagnostic code paths are under code review.
Regressions are caught, and improvements always show up in code review.
To ensure that characterization testing doesn't make it harder to intentionally change these interfaces, there always must be an easy way to regenerate the expected output, as we do with `_NIX_TEST_ACCEPT=1`.
## Integration tests
The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute.

10
doc/manual/src/language/advanced-attributes.md
View file

@ -320,16 +320,6 @@ Derivations can declare some infrequently used optional attributes.
```
- [`unsafeDiscardReferences`]{#adv-attr-unsafeDiscardReferences}\
> **Warning**
> This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md).
>
> To use this attribute, you must enable the
> [`discard-references`](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) experimental feature.
> For example, in [nix.conf](../command-ref/conf-file.md) you could add:
>
> ```
> extra-experimental-features = discard-references
> ```
When using [structured attributes](#adv-attr-structuredAttrs), the
attribute `unsafeDiscardReferences` is an attribute set with a boolean value for each output name.

39
doc/manual/src/language/constructs.md
View file

@ -92,10 +92,10 @@ In this fragment from `all-packages.nix`,
```nix
graphviz = (import ../tools/graphics/graphviz) {
inherit fetchurl stdenv libpng libjpeg expat x11 yacc;
inherit (xlibs) libXaw;
inherit (xorg) libXaw;
};
xlibs = {
xorg = {
libX11 = ...;
libXaw = ...;
...
@ -109,7 +109,7 @@ libjpg = ...;
the set used in the function call to the function defined in
`../tools/graphics/graphviz` inherits a number of variables from the
surrounding scope (`fetchurl` ... `yacc`), but also inherits `libXaw`
(the X Athena Widgets) from the `xlibs` (X11 client-side libraries) set.
(the X Athena Widgets) from the `xorg` set.
Summarizing the fragment
@ -209,29 +209,40 @@ three kinds of patterns:
{ x, y, z, ... } @ args: z + y + x + args.a
```
Here `args` is bound to the entire argument, which is further
matched against the pattern `{ x, y, z,
... }`. `@`-pattern makes mainly sense with an ellipsis(`...`) as
Here `args` is bound to the argument *as passed*, which is further
matched against the pattern `{ x, y, z, ... }`.
The `@`-pattern makes mainly sense with an ellipsis(`...`) as
you can access attribute names as `a`, using `args.a`, which was
given as an additional attribute to the function.
> **Warning**
>
> The `args@` expression is bound to the argument passed to the
> function which means that attributes with defaults that aren't
> explicitly specified in the function call won't cause an
> evaluation error, but won't exist in `args`.
> `args@` binds the name `args` to the attribute set that is passed to the function.
> In particular, `args` does *not* include any default values specified with `?` in the function's set pattern.
>
> For instance
>
> ```nix
> let
> function = args@{ a ? 23, ... }: args;
> f = args@{ a ? 23, ... }: [ a args ];
> in
> function {}
> ````
> f {}
> ```
>
> will evaluate to an empty attribute set.
> is equivalent to
>
> ```nix
> let
> f = args @ { ... }: [ (args.a or 23) args ];
> in
> f {}
> ```
>
> and both expressions will evaluate to:
>
> ```nix
> [ 23 {} ]
> ```
Note that functions do not have names. If you want to give them a name,
you can bind them to an attribute, e.g.,

2
doc/manual/src/package-management/basic-package-mgmt.md
View file

@ -25,7 +25,7 @@ or completely new ones.)
You can manually download the latest version of Nixpkgs from
<https://github.com/NixOS/nixpkgs>. However, it’s much more
convenient to use the Nixpkgs [*channel*](channels.md), since it makes
convenient to use the Nixpkgs [*channel*](../command-ref/nix-channel.md), since it makes
it easy to stay up to date with new versions of Nixpkgs. Nixpkgs is
automatically added to your list of “subscribed” channels when you
install Nix. If this is not the case for some reason, you can add it

50
doc/manual/src/package-management/channels.md
View file

@ -1,50 +0,0 @@
# Channels
If you want to stay up to date with a set of packages, it’s not very
convenient to manually download the latest set of Nix expressions for
those packages and upgrade using `nix-env`. Fortunately, there’s a
better way: *Nix channels*.
A Nix channel is just a URL that points to a place that contains a set
of Nix expressions and a manifest. Using the command
[`nix-channel`](../command-ref/nix-channel.md) you can automatically
stay up to date with whatever is available at that URL.
To see the list of official NixOS channels, visit
<https://nixos.org/channels>.
You can “subscribe” to a channel using `nix-channel --add`, e.g.,
```console
$ nix-channel --add https://nixos.org/channels/nixpkgs-unstable
```
subscribes you to a channel that always contains that latest version of
the Nix Packages collection. (Subscribing really just means that the URL
is added to the file `~/.nix-channels`, where it is read by subsequent
calls to `nix-channel
--update`.) You can “unsubscribe” using `nix-channel
--remove`:
```console
$ nix-channel --remove nixpkgs
```
To obtain the latest Nix expressions available in a channel, do
```console
$ nix-channel --update
```
This downloads and unpacks the Nix expressions in every channel
(downloaded from `url/nixexprs.tar.bz2`). It also makes the union of
each channel’s Nix expressions available by default to `nix-env`
operations (via the symlink `~/.nix-defexpr/channels`). Consequently,
you can then say
```console
$ nix-env --upgrade
```
to upgrade all packages in your profile to the latest versions available
in the subscribed channels.

42
doc/manual/src/release-notes/rl-2.17.md Normal file
View file

@ -0,0 +1,42 @@
# Release 2.17 (2023-07-24)
* [`nix-channel`](../command-ref/nix-channel.md) now supports a `--list-generations` subcommand.
* The function [`builtins.fetchClosure`](../language/builtins.md#builtins-fetchClosure) can now fetch input-addressed paths in [pure evaluation mode](../command-ref/conf-file.md#conf-pure-eval), as those are not impure.
* Nix now allows unprivileged/[`allowed-users`](../command-ref/conf-file.md#conf-allowed-users) to sign paths.
Previously, only [`trusted-users`](../command-ref/conf-file.md#conf-trusted-users) users could sign paths.
* Nested dynamic attributes are now merged correctly by the parser. For example:
```nix
{
nested = {
foo = 1;
};
nested = {
${"ba" + "r"} = 2;
};
}
```
This used to silently discard `nested.bar`, but now behaves as one would expect and evaluates to:
```nix
{ nested = { bar = 2; foo = 1; }; }
```
Note that the feature of merging multiple *full declarations* of attribute sets like `nested` in the example is of questionable value.
It allows writing expressions that are very hard to read, for instance when there are many lines of code between two declarations of the same attribute.
This has been around for a long time and is therefore supported for backwards compatibility, but should not be relied upon.
Instead, consider using the *nested attribute path* syntax:
```nix
{
nested.foo = 1;
nested.${"ba" + "r"} = 2;
}
```
* Tarball flakes can now redirect to an "immutable" URL that will be recorded in lock files. This allows the use of "mutable" tarball URLs like `https://example.org/hello/latest.tar.gz` in flakes. See the [tarball fetcher](../protocols/tarball-fetcher.md) for details.

21
doc/manual/src/release-notes/rl-next.md
View file

@ -1,8 +1,21 @@
# Release X.Y (202?-??-??)
- [`nix-channel`](../command-ref/nix-channel.md) now supports a `--list-generations` subcommand
- Two new builtin functions,
[`builtins.parseFlakeRef`](@docroot@/language/builtins.md#builtins-parseFlakeRef)
and
[`builtins.flakeRefToString`](@docroot@/language/builtins.md#builtins-flakeRefToString),
have been added.
These functions are useful for converting between flake references encoded as attribute sets and URLs.
* The function [`builtins.fetchClosure`](../language/builtins.md#builtins-fetchClosure) can now fetch input-addressed paths in [pure evaluation mode](../command-ref/conf-file.md#conf-pure-eval), as those are not impure.
- [`builtins.toJSON`](@docroot@/language/builtins.md#builtins-parseFlakeRef) now prints [--show-trace](@docroot@/command-ref/conf-file.html#conf-show-trace) items for the path in which it finds an evaluation error.
- Nix now allows unprivileged/[`allowed-users`](../command-ref/conf-file.md#conf-allowed-users) to sign paths.
Previously, only [`trusted-users`](../command-ref/conf-file.md#conf-trusted-users) users could sign paths.
- Error messages regarding malformed input to [`derivation add`](@docroot@/command-ref/new-cli/nix3-derivation-add.md) are now clearer and more detailed.
- The `discard-references` feature has been stabilized.
This means that the
[unsafeDiscardReferences](@docroot@/contributing/experimental-features.md#xp-feature-discard-references)
attribute is no longer guarded by an experimental flag and can be used
freely.
- The JSON output for derived paths with are store paths is now a string, not an object with a single `path` field.
This only affects `nix-build --json` when "building" non-derivation things like fetched sources, which is a no-op.

20
mk/lib.mk
View file

@ -10,6 +10,7 @@ bin-scripts :=
noinst-scripts :=
man-pages :=
install-tests :=
install-tests-groups :=
ifdef HOST_OS
HOST_KERNEL = $(firstword $(subst -, ,$(HOST_OS)))
@ -121,7 +122,16 @@ $(foreach script, $(bin-scripts), $(eval $(call install-program-in,$(script),$(b
$(foreach script, $(bin-scripts), $(eval programs-list += $(script)))
$(foreach script, $(noinst-scripts), $(eval programs-list += $(script)))
$(foreach template, $(template-files), $(eval $(call instantiate-template,$(template))))
$(foreach test, $(install-tests), $(eval $(call run-install-test,$(test))))
$(foreach test, $(install-tests), \
$(eval $(call run-install-test,$(test))) \
$(eval installcheck: $(test).test))
$(foreach test-group, $(install-tests-groups), \
$(eval $(call run-install-test-group,$(test-group))) \
$(eval installcheck: $(test-group).test-group) \
$(foreach test, $($(test-group)-tests), \
$(eval $(call run-install-test,$(test))) \
$(eval $(test-group).test-group: $(test).test)))
$(foreach file, $(man-pages), $(eval $(call install-data-in, $(file), $(mandir)/man$(patsubst .%,%,$(suffix $(file))))))
@ -151,6 +161,14 @@ ifdef libs-list
@echo "The following libraries can be built:"
@echo ""
@for i in $(libs-list); do echo " $$i"; done
endif
ifdef install-tests-groups
@echo ""
@echo "The following groups of functional tests can be run:"
@echo ""
@for i in $(install-tests-groups); do echo " $$i.test-group"; done
@echo ""
@echo "(installcheck includes tests in test groups too.)"
endif
@echo ""
@echo "The following variables control the build:"

8
mk/tests.mk
View file

@ -4,8 +4,6 @@ test-deps =
define run-install-test
installcheck: $1.test
.PHONY: $1.test
$1.test: $1 $(test-deps)
@env BASH=$(bash) $(bash) mk/run-test.sh $1 < /dev/null
@ -16,6 +14,12 @@ define run-install-test
endef
define run-install-test-group
.PHONY: $1.test-group
endef
.PHONY: check installcheck
print-top-help += \

2
perl/lib/Nix/Store.xs
View file

@ -294,10 +294,8 @@ SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name)
auto h = Hash::parseAny(hash, parseHashType(algo));
auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
auto path = store()->makeFixedOutputPath(name, FixedOutputInfo {
.hash = {
.method = method,
.hash = h,
},
.references = {},
});
XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0)));

7
src/build-remote/build-remote.cc
View file

@ -322,7 +322,12 @@ connected:
throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg);
} else {
copyClosure(*store, *sshStore, StorePathSet {*drvPath}, NoRepair, NoCheckSigs, substitute);
auto res = sshStore->buildPathsWithResults({ DerivedPath::Built { *drvPath, OutputsSpec::All {} } });
auto res = sshStore->buildPathsWithResults({
DerivedPath::Built {
.drvPath = makeConstantStorePathRef(*drvPath),
.outputs = OutputsSpec::All {},
}
});
// One path to build should produce exactly one build result
assert(res.size() == 1);
optResult = std::move(res[0]);

127
src/libcmd/built-path.cc Normal file
View file

@ -0,0 +1,127 @@
#include "built-path.hh"
#include "derivations.hh"
#include "store-api.hh"
#include <nlohmann/json.hpp>
#include <optional>
namespace nix {
#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \
bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \
{ \
const MY_TYPE* me = this; \
auto fields1 = std::make_tuple<const CHILD_TYPE &, const FIELD_TYPE &>(*me->drvPath, me->FIELD); \
me = &other; \
auto fields2 = std::make_tuple<const CHILD_TYPE &, const FIELD_TYPE &>(*me->drvPath, me->FIELD); \
return fields1 COMPARATOR fields2; \
}
#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \
CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \
CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \
CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <)
#define FIELD_TYPE std::pair<std::string, StorePath>
CMP(SingleBuiltPath, SingleBuiltPathBuilt, output)
#undef FIELD_TYPE
#define FIELD_TYPE std::map<std::string, StorePath>
CMP(SingleBuiltPath, BuiltPathBuilt, outputs)
#undef FIELD_TYPE
#undef CMP
#undef CMP_ONE
StorePath SingleBuiltPath::outPath() const
{
return std::visit(
overloaded{
[](const SingleBuiltPath::Opaque & p) { return p.path; },
[](const SingleBuiltPath::Built & b) { return b.output.second; },
}, raw()
);
}
StorePathSet BuiltPath::outPaths() const
{
return std::visit(
overloaded{
[](const BuiltPath::Opaque & p) { return StorePathSet{p.path}; },
[](const BuiltPath::Built & b) {
StorePathSet res;
for (auto & [_, path] : b.outputs)
res.insert(path);
return res;
},
}, raw()
);
}
nlohmann::json BuiltPath::Built::toJSON(const Store & store) const
{
nlohmann::json res;
res["drvPath"] = drvPath->toJSON(store);
for (const auto & [outputName, outputPath] : outputs) {
res["outputs"][outputName] = store.printStorePath(outputPath);
}
return res;
}
nlohmann::json SingleBuiltPath::Built::toJSON(const Store & store) const
{
nlohmann::json res;
res["drvPath"] = drvPath->toJSON(store);
auto & [outputName, outputPath] = output;
res["output"] = outputName;
res["outputPath"] = store.printStorePath(outputPath);
return res;
}
nlohmann::json SingleBuiltPath::toJSON(const Store & store) const
{
return std::visit([&](const auto & buildable) {
return buildable.toJSON(store);
}, raw());
}
nlohmann::json BuiltPath::toJSON(const Store & store) const
{
return std::visit([&](const auto & buildable) {
return buildable.toJSON(store);
}, raw());
}
RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const
{
RealisedPath::Set res;
std::visit(
overloaded{
[&](const BuiltPath::Opaque & p) { res.insert(p.path); },
[&](const BuiltPath::Built & p) {
auto drvHashes =
staticOutputHashes(store, store.readDerivation(p.drvPath->outPath()));
for (auto& [outputName, outputPath] : p.outputs) {
if (experimentalFeatureSettings.isEnabled(
Xp::CaDerivations)) {
auto drvOutput = get(drvHashes, outputName);
if (!drvOutput)
throw Error(
"the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)",
store.printStorePath(p.drvPath->outPath()), outputName);
auto thisRealisation = store.queryRealisation(
DrvOutput{*drvOutput, outputName});
assert(thisRealisation); // We’ve built it, so we must
// have the realisation
res.insert(*thisRealisation);
} else {
res.insert(outputPath);
}
}
},
},
raw());
return res;
}
}

90
src/libcmd/built-path.hh Normal file
View file

@ -0,0 +1,90 @@
#include "derived-path.hh"
#include "realisation.hh"
namespace nix {
struct SingleBuiltPath;
struct SingleBuiltPathBuilt {
ref<SingleBuiltPath> drvPath;
std::pair<std::string, StorePath> output;
std::string to_string(const Store & store) const;
static SingleBuiltPathBuilt parse(const Store & store, std::string_view, std::string_view);
nlohmann::json toJSON(const Store & store) const;
DECLARE_CMP(SingleBuiltPathBuilt);
};
using _SingleBuiltPathRaw = std::variant<
DerivedPathOpaque,
SingleBuiltPathBuilt
>;
struct SingleBuiltPath : _SingleBuiltPathRaw {
using Raw = _SingleBuiltPathRaw;
using Raw::Raw;
using Opaque = DerivedPathOpaque;
using Built = SingleBuiltPathBuilt;
inline const Raw & raw() const {
return static_cast<const Raw &>(*this);
}
StorePath outPath() const;
static SingleBuiltPath parse(const Store & store, std::string_view);
nlohmann::json toJSON(const Store & store) const;
};
static inline ref<SingleBuiltPath> staticDrv(StorePath drvPath)
{
return make_ref<SingleBuiltPath>(SingleBuiltPath::Opaque { drvPath });
}
/**
* A built derived path with hints in the form of optional concrete output paths.
*
* See 'BuiltPath' for more an explanation.
*/
struct BuiltPathBuilt {
ref<SingleBuiltPath> drvPath;
std::map<std::string, StorePath> outputs;
std::string to_string(const Store & store) const;
static BuiltPathBuilt parse(const Store & store, std::string_view, std::string_view);
nlohmann::json toJSON(const Store & store) const;
DECLARE_CMP(BuiltPathBuilt);
};
using _BuiltPathRaw = std::variant<
DerivedPath::Opaque,
BuiltPathBuilt
>;
/**
* A built path. Similar to a DerivedPath, but enriched with the corresponding
* output path(s).
*/
struct BuiltPath : _BuiltPathRaw {
using Raw = _BuiltPathRaw;
using Raw::Raw;
using Opaque = DerivedPathOpaque;
using Built = BuiltPathBuilt;
inline const Raw & raw() const {
return static_cast<const Raw &>(*this);
}
StorePathSet outPaths() const;
RealisedPath::Set toRealisedPaths(Store & store) const;
nlohmann::json toJSON(const Store & store) const;
};
typedef std::vector<BuiltPath> BuiltPaths;
}

5
src/libcmd/common-eval-args.cc
View file

@ -1,3 +1,4 @@
#include "eval-settings.hh"
#include "common-eval-args.hh"
#include "shared.hh"
#include "filetransfer.hh"
@ -105,7 +106,9 @@ MixEvalArgs::MixEvalArgs()
)",
.category = category,
.labels = {"path"},
.handler = {[&](std::string s) { searchPath.push_back(s); }}
.handler = {[&](std::string s) {
searchPath.elements.emplace_back(SearchPath::Elem::parse(s));
}}
});
addFlag({

3
src/libcmd/common-eval-args.hh
View file

@ -3,6 +3,7 @@
#include "args.hh"
#include "common-args.hh"
#include "search-path.hh"
namespace nix {
@ -19,7 +20,7 @@ struct MixEvalArgs : virtual Args, virtual MixRepair
Bindings * getAutoArgs(EvalState & state);
Strings searchPath;
SearchPath searchPath;
std::optional<std::string> evalStoreUrl;

2
src/libcmd/installable-attr-path.cc
View file

@ -92,7 +92,7 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths()
for (auto & [drvPath, outputs] : byDrvPath)
res.push_back({
.path = DerivedPath::Built {
.drvPath = drvPath,
.drvPath = makeConstantStorePathRef(drvPath),
.outputs = outputs,
},
.info = make_ref<ExtraPathInfoValue>(ExtraPathInfoValue::Value {

15
src/libcmd/installable-derived-path.cc
View file

@ -18,14 +18,7 @@ DerivedPathsWithInfo InstallableDerivedPath::toDerivedPaths()
std::optional<StorePath> InstallableDerivedPath::getStorePath()
{
return std::visit(overloaded {
[&](const DerivedPath::Built & bfd) {
return bfd.drvPath;
},
[&](const DerivedPath::Opaque & bo) {
return bo.path;
},
}, derivedPath.raw());
return derivedPath.getBaseStorePath();
}
InstallableDerivedPath InstallableDerivedPath::parse(
@ -42,7 +35,7 @@ InstallableDerivedPath InstallableDerivedPath::parse(
// Remove this prior to stabilizing the new CLI.
if (storePath.isDerivation()) {
auto oldDerivedPath = DerivedPath::Built {
.drvPath = storePath,
.drvPath = makeConstantStorePathRef(storePath),
.outputs = OutputsSpec::All { },
};
warn(
@ -55,8 +48,10 @@ InstallableDerivedPath InstallableDerivedPath::parse(
},
// If the user did use ^, we just do exactly what is written.
[&](const ExtendedOutputsSpec::Explicit & outputSpec) -> DerivedPath {
auto drv = make_ref<SingleDerivedPath>(SingleDerivedPath::parse(*store, prefix));
drvRequireExperiment(*drv);
return DerivedPath::Built {
.drvPath = store->parseStorePath(prefix),
.drvPath = std::move(drv),
.outputs = outputSpec,
};
},

2
src/libcmd/installable-flake.cc
View file

@ -118,7 +118,7 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths()
return {{
.path = DerivedPath::Built {
.drvPath = std::move(drvPath),
.drvPath = makeConstantStorePathRef(std::move(drvPath)),
.outputs = std::visit(overloaded {
[&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec {
std::set<std::string> outputsToInstall;

3
src/libcmd/installable-value.cc
View file

@ -55,7 +55,8 @@ std::optional<DerivedPathWithInfo> InstallableValue::trySinglePathToDerivedPaths
else if (v.type() == nString) {
return {{
.path = state->coerceToDerivedPath(pos, v, errorCtx),
.path = DerivedPath::fromSingle(
state->coerceToSingleDerivedPath(pos, v, errorCtx)),
.info = make_ref<ExtraPathInfo>(),
}};
}

37
src/libcmd/installables.cc
View file

@ -11,6 +11,7 @@
#include "derivations.hh"
#include "eval-inline.hh"
#include "eval.hh"
#include "eval-settings.hh"
#include "get-drvs.hh"
#include "store-api.hh"
#include "shared.hh"
@ -672,6 +673,30 @@ ref<Installable> SourceExprCommand::parseInstallable(
return installables.front();
}
static SingleBuiltPath getBuiltPath(ref<Store> evalStore, ref<Store> store, const SingleDerivedPath & b)
{
return std::visit(
overloaded{
[&](const SingleDerivedPath::Opaque & bo) -> SingleBuiltPath {
return SingleBuiltPath::Opaque { bo.path };
},
[&](const SingleDerivedPath::Built & bfd) -> SingleBuiltPath {
auto drvPath = getBuiltPath(evalStore, store, *bfd.drvPath);
// Resolving this instead of `bfd` will yield the same result, but avoid duplicative work.
SingleDerivedPath::Built truncatedBfd {
.drvPath = makeConstantStorePathRef(drvPath.outPath()),
.output = bfd.output,
};
auto outputPath = resolveDerivedPath(*store, truncatedBfd, &*evalStore);
return SingleBuiltPath::Built {
.drvPath = make_ref<SingleBuiltPath>(std::move(drvPath)),
.output = { bfd.output, outputPath },
};
},
},
b.raw());
}
std::vector<BuiltPathWithResult> Installable::build(
ref<Store> evalStore,
ref<Store> store,
@ -725,7 +750,10 @@ std::vector<std::pair<ref<Installable>, BuiltPathWithResult>> Installable::build
[&](const DerivedPath::Built & bfd) {
auto outputs = resolveDerivedPath(*store, bfd, &*evalStore);
res.push_back({aux.installable, {
.path = BuiltPath::Built { bfd.drvPath, outputs },
.path = BuiltPath::Built {
.drvPath = make_ref<SingleBuiltPath>(getBuiltPath(evalStore, store, *bfd.drvPath)),
.outputs = outputs,
},
.info = aux.info}});
},
[&](const DerivedPath::Opaque & bo) {
@ -754,7 +782,10 @@ std::vector<std::pair<ref<Installable>, BuiltPathWithResult>> Installable::build
for (auto & [outputName, realisation] : buildResult.builtOutputs)
outputs.emplace(outputName, realisation.outPath);
res.push_back({aux.installable, {
.path = BuiltPath::Built { bfd.drvPath, outputs },
.path = BuiltPath::Built {
.drvPath = make_ref<SingleBuiltPath>(getBuiltPath(evalStore, store, *bfd.drvPath)),
.outputs = outputs,
},
.info = aux.info,
.result = buildResult}});
},
@ -848,7 +879,7 @@ StorePathSet Installable::toDerivations(
: throw Error("argument '%s' did not evaluate to a derivation", i->what()));
},
[&](const DerivedPath::Built & bfd) {
drvPaths.insert(bfd.drvPath);
drvPaths.insert(resolveDerivedPath(*store, *bfd.drvPath));
},
}, b.path.raw());

1
src/libcmd/installables.hh
View file

@ -5,6 +5,7 @@
#include "path.hh"
#include "outputs-spec.hh"
#include "derived-path.hh"
#include "built-path.hh"
#include "store-api.hh"
#include "build-result.hh"

11
src/libcmd/repl.cc
View file

@ -26,6 +26,7 @@ extern "C" {
#include "eval.hh"
#include "eval-cache.hh"
#include "eval-inline.hh"
#include "eval-settings.hh"
#include "attr-path.hh"
#include "store-api.hh"
#include "log-store.hh"
@ -68,7 +69,7 @@ struct NixRepl
const Path historyFile;
NixRepl(const Strings & searchPath, nix::ref<Store> store,ref<EvalState> state,
NixRepl(const SearchPath & searchPath, nix::ref<Store> store,ref<EvalState> state,
std::function<AnnotatedValues()> getValues);
virtual ~NixRepl();
@ -104,7 +105,7 @@ std::string removeWhitespace(std::string s)
}
NixRepl::NixRepl(const Strings & searchPath, nix::ref<Store> store, ref<EvalState> state,
NixRepl::NixRepl(const SearchPath & searchPath, nix::ref<Store> store, ref<EvalState> state,
std::function<NixRepl::AnnotatedValues()> getValues)
: AbstractNixRepl(state)
, debugTraceIndex(0)
@ -647,7 +648,7 @@ bool NixRepl::processLine(std::string line)
if (command == ":b" || command == ":bl") {
state->store->buildPaths({
DerivedPath::Built {
.drvPath = drvPath,
.drvPath = makeConstantStorePathRef(drvPath),
.outputs = OutputsSpec::All { },
},
});
@ -1024,7 +1025,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
std::unique_ptr<AbstractNixRepl> AbstractNixRepl::create(
const Strings & searchPath, nix::ref<Store> store, ref<EvalState> state,
const SearchPath & searchPath, nix::ref<Store> store, ref<EvalState> state,
std::function<AnnotatedValues()> getValues)
{
return std::make_unique<NixRepl>(
@ -1044,7 +1045,7 @@ void AbstractNixRepl::runSimple(
NixRepl::AnnotatedValues values;
return values;
};
const Strings & searchPath = {};
SearchPath searchPath = {};
auto repl = std::make_unique<NixRepl>(
searchPath,
openStore(),

2
src/libcmd/repl.hh
View file

@ -25,7 +25,7 @@ struct AbstractNixRepl
typedef std::vector<std::pair<Value*,std::string>> AnnotatedValues;
static std::unique_ptr<AbstractNixRepl> create(
const Strings & searchPath, nix::ref<Store> store, ref<EvalState> state,
const SearchPath & searchPath, nix::ref<Store> store, ref<EvalState> state,
std::function<AnnotatedValues()> getValues);
static void runSimple(

2
src/libexpr/eval-cache.cc
View file

@ -599,7 +599,7 @@ string_t AttrCursor::getStringWithContext()
return d.drvPath;
},
[&](const NixStringContextElem::Built & b) -> const StorePath & {
return b.drvPath;
return b.drvPath->getBaseStorePath();
},
[&](const NixStringContextElem::Opaque & o) -> const StorePath & {
return o.path;

102
src/libexpr/eval-settings.cc Normal file
View file

@ -0,0 +1,102 @@
#include "globals.hh"
#include "profiles.hh"
#include "eval.hh"
#include "eval-settings.hh"
namespace nix {
/* Very hacky way to parse $NIX_PATH, which is colon-separated, but
can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */
static Strings parseNixPath(const std::string & s)
{
Strings res;
auto p = s.begin();
while (p != s.end()) {
auto start = p;
auto start2 = p;
while (p != s.end() && *p != ':') {
if (*p == '=') start2 = p + 1;
++p;
}
if (p == s.end()) {
if (p != start) res.push_back(std::string(start, p));
break;
}
if (*p == ':') {
auto prefix = std::string(start2, s.end());
if (EvalSettings::isPseudoUrl(prefix) || hasPrefix(prefix, "flake:")) {
++p;
while (p != s.end() && *p != ':') ++p;
}
res.push_back(std::string(start, p));
if (p == s.end()) break;
}
++p;
}
return res;
}
EvalSettings::EvalSettings()
{
auto var = getEnv("NIX_PATH");
if (var) nixPath = parseNixPath(*var);
}
Strings EvalSettings::getDefaultNixPath()
{
Strings res;
auto add = [&](const Path & p, const std::string & s = std::string()) {
if (pathAccessible(p)) {
if (s.empty()) {
res.push_back(p);
} else {
res.push_back(s + "=" + p);
}
}
};
if (!evalSettings.restrictEval && !evalSettings.pureEval) {
add(getNixDefExpr() + "/channels");
add(rootChannelsDir() + "/nixpkgs", "nixpkgs");
add(rootChannelsDir());
}
return res;
}
bool EvalSettings::isPseudoUrl(std::string_view s)
{
if (s.compare(0, 8, "channel:") == 0) return true;
size_t pos = s.find("://");
if (pos == std::string::npos) return false;
std::string scheme(s, 0, pos);
return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh";
}
std::string EvalSettings::resolvePseudoUrl(std::string_view url)
{
if (hasPrefix(url, "channel:"))
return "https://nixos.org/channels/" + std::string(url.substr(8)) + "/nixexprs.tar.xz";
else
return std::string(url);
}
EvalSettings evalSettings;
static GlobalConfig::Register rEvalSettings(&evalSettings);
Path getNixDefExpr()
{
return settings.useXDGBaseDirectories
? getStateDir() + "/nix/defexpr"
: getHome() + "/.nix-defexpr";
}
}

103
src/libexpr/eval-settings.hh Normal file
View file

@ -0,0 +1,103 @@
#pragma once
#include "config.hh"
namespace nix {
struct EvalSettings : Config
{
EvalSettings();
static Strings getDefaultNixPath();
static bool isPseudoUrl(std::string_view s);
static std::string resolvePseudoUrl(std::string_view url);
Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation",
"Whether builtin functions that allow executing native code should be enabled."};
Setting<Strings> nixPath{
this, getDefaultNixPath(), "nix-path",
R"(
List of directories to be searched for `<...>` file references
In particular, outside of [pure evaluation mode](#conf-pure-evaluation), this determines the value of
[`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath).
)"};
Setting<bool> restrictEval{
this, false, "restrict-eval",
R"(
If set to `true`, the Nix evaluator will not allow access to any
files outside of the Nix search path (as set via the `NIX_PATH`
environment variable or the `-I` option), or to URIs outside of
[`allowed-uris`](../command-ref/conf-file.md#conf-allowed-uris).
The default is `false`.
)"};
Setting<bool> pureEval{this, false, "pure-eval",
R"(
Pure evaluation mode ensures that the result of Nix expressions is fully determined by explicitly declared inputs, and not influenced by external state:
- Restrict file system and network access to files specified by cryptographic hash
- Disable [`bultins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) and [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime)
)"
};
Setting<bool> enableImportFromDerivation{
this, true, "allow-import-from-derivation",
R"(
By default, Nix allows you to `import` from a derivation, allowing
building at evaluation time. With this option set to false, Nix will
throw an error when evaluating an expression that uses this feature,
allowing users to ensure their evaluation will not require any
builds to take place.
)"};
Setting<Strings> allowedUris{this, {}, "allowed-uris",
R"(
A list of URI prefixes to which access is allowed in restricted
evaluation mode. For example, when set to
`https://github.com/NixOS`, builtin functions such as `fetchGit` are
allowed to access `https://github.com/NixOS/patchelf.git`.
)"};
Setting<bool> traceFunctionCalls{this, false, "trace-function-calls",
R"(
If set to `true`, the Nix evaluator will trace every function call.
Nix will print a log message at the "vomit" level for every function
entrance and function exit.
function-trace entered undefined position at 1565795816999559622
function-trace exited undefined position at 1565795816999581277
function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150
function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684
The `undefined position` means the function call is a builtin.
Use the `contrib/stack-collapse.py` script distributed with the Nix
source code to convert the trace logs in to a format suitable for
`flamegraph.pl`.
)"};
Setting<bool> useEvalCache{this, true, "eval-cache",
"Whether to use the flake evaluation cache."};
Setting<bool> ignoreExceptionsDuringTry{this, false, "ignore-try",
R"(
If set to true, ignore exceptions inside 'tryEval' calls when evaluating nix expressions in
debug mode (using the --debugger flag). By default the debugger will pause on all exceptions.
)"};
Setting<bool> traceVerbose{this, false, "trace-verbose",
"Whether `builtins.traceVerbose` should trace its first argument when evaluated."};
};
extern EvalSettings evalSettings;
/**
* Conventionally part of the default nix path in impure mode.
*/
Path getNixDefExpr();
}

171
src/libexpr/eval.cc
View file

@ -1,4 +1,5 @@
#include "eval.hh"
#include "eval-settings.hh"
#include "hash.hh"
#include "types.hh"
#include "util.hh"
@ -420,44 +421,6 @@ void initGC()
}
/* Very hacky way to parse $NIX_PATH, which is colon-separated, but
can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */
static Strings parseNixPath(const std::string & s)
{
Strings res;
auto p = s.begin();
while (p != s.end()) {
auto start = p;
auto start2 = p;
while (p != s.end() && *p != ':') {
if (*p == '=') start2 = p + 1;
++p;
}
if (p == s.end()) {
if (p != start) res.push_back(std::string(start, p));
break;
}
if (*p == ':') {
auto prefix = std::string(start2, s.end());
if (EvalSettings::isPseudoUrl(prefix) || hasPrefix(prefix, "flake:")) {
++p;
while (p != s.end() && *p != ':') ++p;
}
res.push_back(std::string(start, p));
if (p == s.end()) break;
}
++p;
}
return res;
}
ErrorBuilder & ErrorBuilder::atPos(PosIdx pos)
{
info.errPos = state.positions[pos];
@ -498,7 +461,7 @@ ErrorBuilder & ErrorBuilder::withFrame(const Env & env, const Expr & expr)
EvalState::EvalState(
const Strings & _searchPath,
const SearchPath & _searchPath,
ref<Store> store,
std::shared_ptr<Store> buildStore)
: sWith(symbols.create("<with>"))
@ -563,30 +526,32 @@ EvalState::EvalState(
/* Initialise the Nix expression search path. */
if (!evalSettings.pureEval) {
for (auto & i : _searchPath) addToSearchPath(i);
for (auto & i : evalSettings.nixPath.get()) addToSearchPath(i);
for (auto & i : _searchPath.elements)
addToSearchPath(SearchPath::Elem {i});
for (auto & i : evalSettings.nixPath.get())
addToSearchPath(SearchPath::Elem::parse(i));
}
if (evalSettings.restrictEval || evalSettings.pureEval) {
allowedPaths = PathSet();
for (auto & i : searchPath) {
auto r = resolveSearchPathElem(i);
if (!r.first) continue;
for (auto & i : searchPath.elements) {
auto r = resolveSearchPathPath(i.path);
if (!r) continue;
auto path = r.second;
auto path = *std::move(r);
if (store->isInStore(r.second)) {
if (store->isInStore(path)) {
try {
StorePathSet closure;
store->computeFSClosure(store->toStorePath(r.second).first, closure);
store->computeFSClosure(store->toStorePath(path).first, closure);
for (auto & path : closure)
allowPath(path);
} catch (InvalidPath &) {
allowPath(r.second);
allowPath(path);
}
} else
allowPath(r.second);
allowPath(path);
}
}
@ -1066,17 +1031,18 @@ void EvalState::mkOutputString(
Value & value,
const StorePath & drvPath,
const std::string outputName,
std::optional<StorePath> optOutputPath)
std::optional<StorePath> optOutputPath,
const ExperimentalFeatureSettings & xpSettings)
{
value.mkString(
optOutputPath
? store->printStorePath(*std::move(optOutputPath))
/* Downstream we would substitute this for an actual path once
we build the floating CA derivation */
: DownstreamPlaceholder::unknownCaOutput(drvPath, outputName).render(),
: DownstreamPlaceholder::unknownCaOutput(drvPath, outputName, xpSettings).render(),
NixStringContext {
NixStringContextElem::Built {
.drvPath = drvPath,
.drvPath = makeConstantStorePathRef(drvPath),
.output = outputName,
}
});
@ -2333,7 +2299,7 @@ StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringCon
}
std::pair<DerivedPath, std::string_view> EvalState::coerceToDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx)
std::pair<SingleDerivedPath, std::string_view> EvalState::coerceToSingleDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx)
{
NixStringContext context;
auto s = forceString(v, context, pos, errorCtx);
@ -2344,21 +2310,16 @@ std::pair<DerivedPath, std::string_view> EvalState::coerceToDerivedPathUnchecked
s, csize)
.withTrace(pos, errorCtx).debugThrow<EvalError>();
auto derivedPath = std::visit(overloaded {
[&](NixStringContextElem::Opaque && o) -> DerivedPath {
return DerivedPath::Opaque {
.path = std::move(o.path),
};
[&](NixStringContextElem::Opaque && o) -> SingleDerivedPath {
return std::move(o);
},
[&](NixStringContextElem::DrvDeep &&) -> DerivedPath {
[&](NixStringContextElem::DrvDeep &&) -> SingleDerivedPath {
error(
"string '%s' has a context which refers to a complete source and binary closure. This is not supported at this time",
s).withTrace(pos, errorCtx).debugThrow<EvalError>();
},
[&](NixStringContextElem::Built && b) -> DerivedPath {
return DerivedPath::Built {
.drvPath = std::move(b.drvPath),
.outputs = OutputsSpec::Names { std::move(b.output) },
};
[&](NixStringContextElem::Built && b) -> SingleDerivedPath {
return std::move(b);
},
}, ((NixStringContextElem &&) *context.begin()).raw());
return {
@ -2368,12 +2329,12 @@ std::pair<DerivedPath, std::string_view> EvalState::coerceToDerivedPathUnchecked
}
DerivedPath EvalState::coerceToDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx)
SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx)
{
auto [derivedPath, s_] = coerceToDerivedPathUnchecked(pos, v, errorCtx);
auto [derivedPath, s_] = coerceToSingleDerivedPathUnchecked(pos, v, errorCtx);
auto s = s_;
std::visit(overloaded {
[&](const DerivedPath::Opaque & o) {
[&](const SingleDerivedPath::Opaque & o) {
auto sExpected = store->printStorePath(o.path);
if (s != sExpected)
error(
@ -2381,25 +2342,27 @@ DerivedPath EvalState::coerceToDerivedPath(const PosIdx pos, Value & v, std::str
s, sExpected)
.withTrace(pos, errorCtx).debugThrow<EvalError>();
},
[&](const DerivedPath::Built & b) {
// TODO need derived path with single output to make this
// total. Will add as part of RFC 92 work and then this is
// cleaned up.
auto output = *std::get<OutputsSpec::Names>(b.outputs).begin();
auto drv = store->readDerivation(b.drvPath);
auto i = drv.outputs.find(output);
[&](const SingleDerivedPath::Built & b) {
auto sExpected = std::visit(overloaded {
[&](const SingleDerivedPath::Opaque & o) {
auto drv = store->readDerivation(o.path);
auto i = drv.outputs.find(b.output);
if (i == drv.outputs.end())
throw Error("derivation '%s' does not have output '%s'", store->printStorePath(b.drvPath), output);
auto optOutputPath = i->second.path(*store, drv.name, output);
throw Error("derivation '%s' does not have output '%s'", b.drvPath->to_string(*store), b.output);
auto optOutputPath = i->second.path(*store, drv.name, b.output);
// This is testing for the case of CA derivations
auto sExpected = optOutputPath
return optOutputPath
? store->printStorePath(*optOutputPath)
: DownstreamPlaceholder::unknownCaOutput(b.drvPath, output).render();
: DownstreamPlaceholder::fromSingleDerivedPathBuilt(b).render();
},
[&](const SingleDerivedPath::Built & o) {
return DownstreamPlaceholder::fromSingleDerivedPathBuilt(b).render();
},
}, b.drvPath->raw());
if (s != sExpected)
error(
"string '%s' has context with the output '%s' from derivation '%s', but the string is not the right placeholder for this derivation output. It should be '%s'",
s, output, store->printStorePath(b.drvPath), sExpected)
s, b.output, b.drvPath->to_string(*store), sExpected)
.withTrace(pos, errorCtx).debugThrow<EvalError>();
}
}, derivedPath.raw());
@ -2624,54 +2587,4 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) {
}
EvalSettings::EvalSettings()
{
auto var = getEnv("NIX_PATH");
if (var) nixPath = parseNixPath(*var);
}
Strings EvalSettings::getDefaultNixPath()
{
Strings res;
auto add = [&](const Path & p, const std::string & s = std::string()) {
if (pathAccessible(p)) {
if (s.empty()) {
res.push_back(p);
} else {
res.push_back(s + "=" + p);
}
}
};
if (!evalSettings.restrictEval && !evalSettings.pureEval) {
add(settings.useXDGBaseDirectories ? getStateDir() + "/nix/defexpr/channels" : getHome() + "/.nix-defexpr/channels");
add(rootChannelsDir() + "/nixpkgs", "nixpkgs");
add(rootChannelsDir());
}
return res;
}
bool EvalSettings::isPseudoUrl(std::string_view s)
{
if (s.compare(0, 8, "channel:") == 0) return true;
size_t pos = s.find("://");
if (pos == std::string::npos) return false;
std::string scheme(s, 0, pos);
return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh";
}
std::string EvalSettings::resolvePseudoUrl(std::string_view url)
{
if (hasPrefix(url, "channel:"))
return "https://nixos.org/channels/" + std::string(url.substr(8)) + "/nixexprs.tar.xz";
else
return std::string(url);
}
EvalSettings evalSettings;
static GlobalConfig::Register rEvalSettings(&evalSettings);
}

131
src/libexpr/eval.hh
View file

@ -9,6 +9,7 @@
#include "config.hh"
#include "experimental-features.hh"
#include "input-accessor.hh"
#include "search-path.hh"
#include <map>
#include <optional>
@ -21,7 +22,7 @@ namespace nix {
class Store;
class EvalState;
class StorePath;
struct DerivedPath;
struct SingleDerivedPath;
enum RepairFlag : bool;
@ -122,15 +123,6 @@ std::string printValue(const EvalState & state, const Value & v);
std::ostream & operator << (std::ostream & os, const ValueType t);
struct SearchPathElem
{
std::string prefix;
// FIXME: maybe change this to an std::variant<SourcePath, URL>.
std::string path;
};
typedef std::list<SearchPathElem> SearchPath;
/**
* Initialise the Boehm GC, if applicable.
*/
@ -317,7 +309,7 @@ private:
SearchPath searchPath;
std::map<std::string, std::pair<bool, std::string>> searchPathResolved;
std::map<std::string, std::optional<std::string>> searchPathResolved;
/**
* Cache used by checkSourcePath().
@ -344,12 +336,12 @@ private:
public:
EvalState(
const Strings & _searchPath,
const SearchPath & _searchPath,
ref<Store> store,
std::shared_ptr<Store> buildStore = nullptr);
~EvalState();
void addToSearchPath(const std::string & s);
void addToSearchPath(SearchPath::Elem && elem);
SearchPath getSearchPath() { return searchPath; }
@ -431,12 +423,16 @@ public:
* Look up a file in the search path.
*/
SourcePath findFile(const std::string_view path);
SourcePath findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos);
SourcePath findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos);
/**
* Try to resolve a search path value (not the optinal key part)
*
* If the specified search path element is a URI, download it.
*
* If it is not found, return `std::nullopt`
*/
std::pair<bool, std::string> resolveSearchPathElem(const SearchPathElem & elem);
std::optional<std::string> resolveSearchPathPath(const SearchPath::Path & path);
/**
* Evaluate an expression to normal form
@ -536,12 +532,12 @@ public:
StorePath coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx);
/**
* Part of `coerceToDerivedPath()` without any store IO which is exposed for unit testing only.
* Part of `coerceToSingleDerivedPath()` without any store IO which is exposed for unit testing only.
*/
std::pair<DerivedPath, std::string_view> coerceToDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx);
std::pair<SingleDerivedPath, std::string_view> coerceToSingleDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx);
/**
* Coerce to `DerivedPath`.
* Coerce to `SingleDerivedPath`.
*
* Must be a string which is either a literal store path or a
* "placeholder (see `DownstreamPlaceholder`).
@ -555,7 +551,7 @@ public:
* source of truth, and ultimately tells us what we want, and then
* we ensure the string corresponds to it.
*/
DerivedPath coerceToDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx);
SingleDerivedPath coerceToSingleDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx);
public:
@ -693,12 +689,15 @@ public:
* be passed if and only if output store object is input-addressed.
* Will be printed to form string if passed, otherwise a placeholder
* will be used (see `DownstreamPlaceholder`).
*
* @param xpSettings Stop-gap to avoid globals during unit tests.
*/
void mkOutputString(
Value & value,
const StorePath & drvPath,
const std::string outputName,
std::optional<StorePath> optOutputPath);
std::optional<StorePath> optOutputPath,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx);
@ -791,98 +790,6 @@ struct InvalidPathError : EvalError
#endif
};
struct EvalSettings : Config
{
EvalSettings();
static Strings getDefaultNixPath();
static bool isPseudoUrl(std::string_view s);
static std::string resolvePseudoUrl(std::string_view url);
Setting<bool> enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation",
"Whether builtin functions that allow executing native code should be enabled."};
Setting<Strings> nixPath{
this, getDefaultNixPath(), "nix-path",
R"(
List of directories to be searched for `<...>` file references
In particular, outside of [pure evaluation mode](#conf-pure-evaluation), this determines the value of
[`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtin-constants-nixPath).
)"};
Setting<bool> restrictEval{
this, false, "restrict-eval",
R"(
If set to `true`, the Nix evaluator will not allow access to any
files outside of the Nix search path (as set via the `NIX_PATH`
environment variable or the `-I` option), or to URIs outside of
[`allowed-uris`](../command-ref/conf-file.md#conf-allowed-uris).
The default is `false`.
)"};
Setting<bool> pureEval{this, false, "pure-eval",
R"(
Pure evaluation mode ensures that the result of Nix expressions is fully determined by explicitly declared inputs, and not influenced by external state:
- Restrict file system and network access to files specified by cryptographic hash
- Disable [`bultins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) and [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime)
)"
};
Setting<bool> enableImportFromDerivation{
this, true, "allow-import-from-derivation",
R"(
By default, Nix allows you to `import` from a derivation, allowing
building at evaluation time. With this option set to false, Nix will
throw an error when evaluating an expression that uses this feature,
allowing users to ensure their evaluation will not require any
builds to take place.
)"};
Setting<Strings> allowedUris{this, {}, "allowed-uris",
R"(
A list of URI prefixes to which access is allowed in restricted
evaluation mode. For example, when set to
`https://github.com/NixOS`, builtin functions such as `fetchGit` are
allowed to access `https://github.com/NixOS/patchelf.git`.
)"};
Setting<bool> traceFunctionCalls{this, false, "trace-function-calls",
R"(
If set to `true`, the Nix evaluator will trace every function call.
Nix will print a log message at the "vomit" level for every function
entrance and function exit.
function-trace entered undefined position at 1565795816999559622
function-trace exited undefined position at 1565795816999581277
function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150
function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684
The `undefined position` means the function call is a builtin.
Use the `contrib/stack-collapse.py` script distributed with the Nix
source code to convert the trace logs in to a format suitable for
`flamegraph.pl`.
)"};
Setting<bool> useEvalCache{this, true, "eval-cache",
"Whether to use the flake evaluation cache."};
Setting<bool> ignoreExceptionsDuringTry{this, false, "ignore-try",
R"(
If set to true, ignore exceptions inside 'tryEval' calls when evaluating nix expressions in
debug mode (using the --debugger flag). By default the debugger will pause on all exceptions.
)"};
Setting<bool> traceVerbose{this, false, "trace-verbose",
"Whether `builtins.traceVerbose` should trace its first argument when evaluated."};
};
extern EvalSettings evalSettings;
static const std::string corepkgsPrefix{"/__corepkgs__/"};
template<class ErrorType>

96
src/libexpr/flake/flake.cc
View file

@ -1,5 +1,6 @@
#include "flake.hh"
#include "eval.hh"
#include "eval-settings.hh"
#include "lockfile.hh"
#include "primops.hh"
#include "eval-inline.hh"
@ -793,6 +794,101 @@ static RegisterPrimOp r2({
.experimentalFeature = Xp::Flakes,
});
static void prim_parseFlakeRef(
EvalState & state,
const PosIdx pos,
Value * * args,
Value & v)
{
std::string flakeRefS(state.forceStringNoCtx(*args[0], pos,
"while evaluating the argument passed to builtins.parseFlakeRef"));
auto attrs = parseFlakeRef(flakeRefS, {}, true).toAttrs();
auto binds = state.buildBindings(attrs.size());
for (const auto & [key, value] : attrs) {
auto s = state.symbols.create(key);
auto & vv = binds.alloc(s);
std::visit(overloaded {
[&vv](const std::string & value) { vv.mkString(value); },
[&vv](const uint64_t & value) { vv.mkInt(value); },
[&vv](const Explicit<bool> & value) { vv.mkBool(value.t); }
}, value);
}
v.mkAttrs(binds);
}
static RegisterPrimOp r3({
.name = "__parseFlakeRef",
.args = {"flake-ref"},
.doc = R"(
Parse a flake reference, and return its exploded form.
For example:
```nix
builtins.parseFlakeRef "github:NixOS/nixpkgs/23.05?dir=lib"
```
evaluates to:
```nix
{ dir = "lib"; owner = "NixOS"; ref = "23.05"; repo = "nixpkgs"; type = "github"; }
```
)",
.fun = prim_parseFlakeRef,
.experimentalFeature = Xp::Flakes,
});
static void prim_flakeRefToString(
EvalState & state,
const PosIdx pos,
Value * * args,
Value & v)
{
state.forceAttrs(*args[0], noPos,
"while evaluating the argument passed to builtins.flakeRefToString");
fetchers::Attrs attrs;
for (const auto & attr : *args[0]->attrs) {
auto t = attr.value->type();
if (t == nInt) {
attrs.emplace(state.symbols[attr.name],
(uint64_t) attr.value->integer);
} else if (t == nBool) {
attrs.emplace(state.symbols[attr.name],
Explicit<bool> { attr.value->boolean });
} else if (t == nString) {
attrs.emplace(state.symbols[attr.name],
std::string(attr.value->str()));
} else {
state.error(
"flake reference attribute sets may only contain integers, Booleans, "
"and strings, but attribute '%s' is %s",
state.symbols[attr.name],
showType(*attr.value)).debugThrow<EvalError>();
}
}
auto flakeRef = FlakeRef::fromAttrs(attrs);
v.mkString(flakeRef.to_string());
}
static RegisterPrimOp r4({
.name = "__flakeRefToString",
.args = {"attrs"},
.doc = R"(
Convert a flake reference from attribute set format to URL format.
For example:
```nix
builtins.flakeRefToString {
dir = "lib"; owner = "NixOS"; ref = "23.05"; repo = "nixpkgs"; type = "github";
}
```
evaluates to
```nix
"github:NixOS/nixpkgs/23.05?dir=lib"
```
)",
.fun = prim_flakeRefToString,
.experimentalFeature = Xp::Flakes,
});
}
Fingerprint LockedFlake::getFingerprint() const

6
src/libexpr/flake/flakeref.cc
View file

@ -105,7 +105,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
};
return std::make_pair(
FlakeRef(Input::fromURL(parsedURL), ""),
FlakeRef(Input::fromURL(parsedURL, isFlake), ""),
percentDecode(match.str(6)));
}
@ -176,7 +176,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
parsedURL.query.insert_or_assign("shallow", "1");
return std::make_pair(
FlakeRef(Input::fromURL(parsedURL), getOr(parsedURL.query, "dir", "")),
FlakeRef(Input::fromURL(parsedURL, isFlake), getOr(parsedURL.query, "dir", "")),
fragment);
}
@ -204,7 +204,7 @@ std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
std::string fragment;
std::swap(fragment, parsedURL.fragment);
auto input = Input::fromURL(parsedURL);
auto input = Input::fromURL(parsedURL, isFlake);
input.parent = baseDir;
return std::make_pair(

86
src/libexpr/parser.y
View file

@ -22,6 +22,7 @@
#include "nixexpr.hh"
#include "eval.hh"
#include "eval-settings.hh"
#include "globals.hh"
namespace nix {
@ -137,6 +138,7 @@ static void addAttr(ExprAttrs * attrs, AttrPath && attrPath,
dupAttr(state, ad.first, j2->second.pos, ad.second.pos);
jAttrs->attrs.emplace(ad.first, ad.second);
}
jAttrs->dynamicAttrs.insert(jAttrs->dynamicAttrs.end(), ae->dynamicAttrs.begin(), ae->dynamicAttrs.end());
} else {
dupAttr(state, attrPath, pos, j->second.pos);
}
@ -736,22 +738,9 @@ Expr * EvalState::parseStdin()
}
void EvalState::addToSearchPath(const std::string & s)
void EvalState::addToSearchPath(SearchPath::Elem && elem)
{
size_t pos = s.find('=');
std::string prefix;
Path path;
if (pos == std::string::npos) {
path = s;
} else {
prefix = std::string(s, 0, pos);
path = std::string(s, pos + 1);
}
searchPath.emplace_back(SearchPathElem {
.prefix = prefix,
.path = path,
});
searchPath.elements.emplace_back(std::move(elem));
}
@ -761,22 +750,19 @@ SourcePath EvalState::findFile(const std::string_view path)
}
SourcePath EvalState::findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos)
SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos)
{
for (auto & i : searchPath) {
std::string suffix;
if (i.prefix.empty())
suffix = concatStrings("/", path);
else {
auto s = i.prefix.size();
if (path.compare(0, s, i.prefix) != 0 ||
(path.size() > s && path[s] != '/'))
continue;
suffix = path.size() == s ? "" : concatStrings("/", path.substr(s));
}
auto r = resolveSearchPathElem(i);
if (!r.first) continue;
Path res = r.second + suffix;
for (auto & i : searchPath.elements) {
auto suffixOpt = i.prefix.suffixIfPotentialMatch(path);
if (!suffixOpt) continue;
auto suffix = *suffixOpt;
auto rOpt = resolveSearchPathPath(i.path);
if (!rOpt) continue;
auto r = *rOpt;
Path res = suffix == "" ? r : concatStrings(r, "/", suffix);
if (pathExists(res)) return CanonPath(canonPath(res));
}
@ -793,49 +779,53 @@ SourcePath EvalState::findFile(SearchPath & searchPath, const std::string_view p
}
std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathElem & elem)
std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Path & value0)
{
auto i = searchPathResolved.find(elem.path);
auto & value = value0.s;
auto i = searchPathResolved.find(value);
if (i != searchPathResolved.end()) return i->second;
std::pair<bool, std::string> res;
std::optional<std::string> res;
if (EvalSettings::isPseudoUrl(elem.path)) {
if (EvalSettings::isPseudoUrl(value)) {
try {
auto storePath = fetchers::downloadTarball(
store, EvalSettings::resolvePseudoUrl(elem.path), "source", false).tree.storePath;
res = { true, store->toRealPath(storePath) };
store, EvalSettings::resolvePseudoUrl(value), "source", false).tree.storePath;
res = { store->toRealPath(storePath) };
} catch (FileTransferError & e) {
logWarning({
.msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.path)
.msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value)
});
res = { false, "" };
res = std::nullopt;
}
}
else if (hasPrefix(elem.path, "flake:")) {
else if (hasPrefix(value, "flake:")) {
experimentalFeatureSettings.require(Xp::Flakes);
auto flakeRef = parseFlakeRef(elem.path.substr(6), {}, true, false);
debug("fetching flake search path element '%s''", elem.path);
auto flakeRef = parseFlakeRef(value.substr(6), {}, true, false);
debug("fetching flake search path element '%s''", value);
auto storePath = flakeRef.resolve(store).fetchTree(store).first.storePath;
res = { true, store->toRealPath(storePath) };
res = { store->toRealPath(storePath) };
}
else {
auto path = absPath(elem.path);
auto path = absPath(value);
if (pathExists(path))
res = { true, path };
res = { path };
else {
logWarning({
.msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", elem.path)
.msg = hintfmt("Nix search path entry '%1%' does not exist, ignoring", value)
});
res = { false, "" };
res = std::nullopt;
}
}
debug("resolved search path element '%s' to '%s'", elem.path, res.second);
if (res)
debug("resolved search path element '%s' to '%s'", value, *res);
else
debug("failed to resolve search path element '%s'", value);
searchPathResolved[elem.path] = res;
searchPathResolved[value] = res;
return res;
}

56
src/libexpr/primops.cc
View file

@ -3,6 +3,7 @@
#include "downstream-placeholder.hh"
#include "eval-inline.hh"
#include "eval.hh"
#include "eval-settings.hh"
#include "globals.hh"
#include "json-to-value.hh"
#include "names.hh"
@ -55,7 +56,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
.drvPath = b.drvPath,
.outputs = OutputsSpec::Names { b.output },
});
ensureValid(b.drvPath);
ensureValid(b.drvPath->getBaseStorePath());
},
[&](const NixStringContextElem::Opaque & o) {
auto ctxS = store->printStorePath(o.path);
@ -76,29 +77,32 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
if (!evalSettings.enableImportFromDerivation)
debugThrowLastTrace(Error(
"cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled",
store->printStorePath(drvs.begin()->drvPath)));
drvs.begin()->to_string(*store)));
/* Build/substitute the context. */
std::vector<DerivedPath> buildReqs;
for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d });
store->buildPaths(buildReqs);
/* Get all the output paths corresponding to the placeholders we had */
for (auto & drv : drvs) {
auto outputs = resolveDerivedPath(*store, drv);
for (auto & [outputName, outputPath] : outputs) {
res.insert_or_assign(
DownstreamPlaceholder::unknownCaOutput(drv.drvPath, outputName).render(),
store->printStorePath(outputPath)
);
}
}
/* Add the output of this derivations to the allowed
paths. */
if (allowedPaths) {
for (auto & [_placeholder, outputPath] : res) {
allowPath(store->toRealPath(outputPath));
allowPath(outputPath);
}
/* Get all the output paths corresponding to the placeholders we had */
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
res.insert_or_assign(
DownstreamPlaceholder::fromSingleDerivedPathBuilt(
SingleDerivedPath::Built {
.drvPath = drv.drvPath,
.output = outputName,
}).render(),
store->printStorePath(outputPath)
);
}
}
}
@ -1251,7 +1255,10 @@ drvName, Bindings * attrs, Value & v)
}
},
[&](const NixStringContextElem::Built & b) {
drv.inputDrvs[b.drvPath].insert(b.output);
if (auto * p = std::get_if<DerivedPath::Opaque>(&*b.drvPath))
drv.inputDrvs[p->path].insert(b.output);
else
throw UnimplementedError("Dependencies on the outputs of dynamic derivations are not yet supported");
},
[&](const NixStringContextElem::Opaque & o) {
drv.inputSrcs.insert(o.path);
@ -1300,9 +1307,10 @@ drvName, Bindings * attrs, Value & v)
auto method = ingestionMethod.value_or(FileIngestionMethod::Flat);
DerivationOutput::CAFixed dof {
.ca = ContentAddress::fromParts(
std::move(method),
std::move(h)),
.ca = ContentAddress {
.method = std::move(method),
.hash = std::move(h),
},
};
drv.env["out"] = state.store->printStorePath(dof.path(*state.store, drvName, "out"));
@ -1658,9 +1666,9 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V
}));
}
searchPath.emplace_back(SearchPathElem {
.prefix = prefix,
.path = path,
searchPath.elements.emplace_back(SearchPath::Elem {
.prefix = SearchPath::Prefix { .s = prefix },
.path = SearchPath::Path { .s = path },
});
}
@ -2164,10 +2172,8 @@ static void addPath(
std::optional<StorePath> expectedStorePath;
if (expectedHash)
expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo {
.hash = {
.method = method,
.hash = *expectedHash,
},
.references = {},
});
@ -4329,12 +4335,12 @@ void EvalState::createBaseEnv()
});
/* Add a value containing the current Nix expression search path. */
mkList(v, searchPath.size());
mkList(v, searchPath.elements.size());
int n = 0;
for (auto & i : searchPath) {
for (auto & i : searchPath.elements) {
auto attrs = buildBindings(2);
attrs.alloc("path").mkString(i.path);
attrs.alloc("prefix").mkString(i.prefix);
attrs.alloc("path").mkString(i.path.s);
attrs.alloc("prefix").mkString(i.prefix.s);
(v.listElems()[n++] = allocValue())->mkAttrs(attrs);
}
addConstant("__nixPath", v, {

7
src/libexpr/primops/context.cc
View file

@ -106,7 +106,10 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args,
contextInfos[std::move(d.drvPath)].allOutputs = true;
},
[&](NixStringContextElem::Built && b) {
contextInfos[std::move(b.drvPath)].outputs.emplace_back(std::move(b.output));
// FIXME should eventually show string context as is, no
// resolving here.
auto drvPath = resolveDerivedPath(*state.store, *b.drvPath);
contextInfos[std::move(drvPath)].outputs.emplace_back(std::move(b.output));
},
[&](NixStringContextElem::Opaque && o) {
contextInfos[std::move(o.path)].path = true;
@ -222,7 +225,7 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar
for (auto elem : iter->value->listItems()) {
auto outputName = state.forceStringNoCtx(*elem, iter->pos, "while evaluating an output name within a string context");
context.emplace(NixStringContextElem::Built {
.drvPath = namePath,
.drvPath = makeConstantStorePathRef(namePath),
.output = std::string { outputName },
});
}

1
src/libexpr/primops/fetchMercurial.cc
View file

@ -1,5 +1,6 @@
#include "primops.hh"
#include "eval-inline.hh"
#include "eval-settings.hh"
#include "store-api.hh"
#include "fetchers.hh"
#include "url.hh"

3
src/libexpr/primops/fetchTree.cc
View file

@ -1,5 +1,6 @@
#include "primops.hh"
#include "eval-inline.hh"
#include "eval-settings.hh"
#include "store-api.hh"
#include "fetchers.hh"
#include "filetransfer.hh"
@ -254,10 +255,8 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
auto expectedPath = state.store->makeFixedOutputPath(
name,
FixedOutputInfo {
.hash = {
.method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat,
.hash = *expectedHash,
},
.references = {}
});

56
src/libexpr/search-path.cc Normal file
View file

@ -0,0 +1,56 @@
#include "search-path.hh"
#include "util.hh"
namespace nix {
std::optional<std::string_view> SearchPath::Prefix::suffixIfPotentialMatch(
std::string_view path) const
{
auto n = s.size();
/* Non-empty prefix and suffix must be separated by a /, or the
prefix is not a valid path prefix. */
bool needSeparator = n > 0 && (path.size() - n) > 0;
if (needSeparator && path[n] != '/') {
return std::nullopt;
}
/* Prefix must be prefix of this path. */
if (path.compare(0, n, s) != 0) {
return std::nullopt;
}
/* Skip next path separator. */
return {
path.substr(needSeparator ? n + 1 : n)
};
}
SearchPath::Elem SearchPath::Elem::parse(std::string_view rawElem)
{
size_t pos = rawElem.find('=');
return SearchPath::Elem {
.prefix = Prefix {
.s = pos == std::string::npos
? std::string { "" }
: std::string { rawElem.substr(0, pos) },
},
.path = Path {
.s = std::string { rawElem.substr(pos + 1) },
},
};
}
SearchPath parseSearchPath(const Strings & rawElems)
{
SearchPath res;
for (auto & rawElem : rawElems)
res.elements.emplace_back(SearchPath::Elem::parse(rawElem));
return res;
}
}

108
src/libexpr/search-path.hh Normal file
View file

@ -0,0 +1,108 @@
#pragma once
///@file
#include <optional>
#include "types.hh"
#include "comparator.hh"
namespace nix {
/**
* A "search path" is a list of ways look for something, used with
* `builtins.findFile` and `< >` lookup expressions.
*/
struct SearchPath
{
/**
* A single element of a `SearchPath`.
*
* Each element is tried in succession when looking up a path. The first
* element to completely match wins.
*/
struct Elem;
/**
* The first part of a `SearchPath::Elem` pair.
*
* Called a "prefix" because it takes the form of a prefix of a file
* path (first `n` path components). When looking up a path, to use
* a `SearchPath::Elem`, its `Prefix` must match the path.
*/
struct Prefix;
/**
* The second part of a `SearchPath::Elem` pair.
*
* It is either a path or a URL (with certain restrictions / extra
* structure).
*
* If the prefix of the path we are looking up matches, we then
* check if the rest of the path points to something that exists
* within the directory denoted by this. If so, the
* `SearchPath::Elem` as a whole matches, and that *something* being
* pointed to by the rest of the path we are looking up is the
* result.
*/
struct Path;
/**
* The list of search path elements. Each one is checked for a path
* when looking up. (The actual lookup entry point is in `EvalState`
* not in this class.)
*/
std::list<SearchPath::Elem> elements;
/**
* Parse a string into a `SearchPath`
*/
static SearchPath parse(const Strings & rawElems);
};
struct SearchPath::Prefix
{
/**
* Underlying string
*
* @todo Should we normalize this when constructing a `SearchPath::Prefix`?
*/
std::string s;
GENERATE_CMP(SearchPath::Prefix, me->s);
/**
* If the path possibly matches this search path element, return the
* suffix that we should look for inside the resolved value of the
* element
* Note the double optionality in the name. While we might have a matching prefix, the suffix may not exist.
*/
std::optional<std::string_view> suffixIfPotentialMatch(std::string_view path) const;
};
struct SearchPath::Path
{
/**
* The location of a search path item, as a path or URL.
*
* @todo Maybe change this to `std::variant<SourcePath, URL>`.
*/
std::string s;
GENERATE_CMP(SearchPath::Path, me->s);
};
struct SearchPath::Elem
{
Prefix prefix;
Path path;
GENERATE_CMP(SearchPath::Elem, me->prefix, me->path);
/**
* Parse a string into a `SearchPath::Elem`
*/
static SearchPath::Elem parse(std::string_view rawElem);
};
}

35
src/libexpr/tests/derived-path.cc
View file

@ -21,12 +21,12 @@ TEST_F(DerivedPathExpressionTest, force_init)
RC_GTEST_FIXTURE_PROP(
DerivedPathExpressionTest,
prop_opaque_path_round_trip,
(const DerivedPath::Opaque & o))
(const SingleDerivedPath::Opaque & o))
{
auto * v = state.allocValue();
state.mkStorePathString(o.path, *v);
auto d = state.coerceToDerivedPath(noPos, *v, "");
RC_ASSERT(DerivedPath { o } == d);
auto d = state.coerceToSingleDerivedPath(noPos, *v, "");
RC_ASSERT(SingleDerivedPath { o } == d);
}
// TODO use DerivedPath::Built for parameter once it supports a single output
@ -37,14 +37,21 @@ RC_GTEST_FIXTURE_PROP(
prop_built_path_placeholder_round_trip,
(const StorePath & drvPath, const StorePathName & outputName))
{
/**
* We set these in tests rather than the regular globals so we don't have
* to worry about race conditions if the tests run concurrently.
*/
ExperimentalFeatureSettings mockXpSettings;
mockXpSettings.set("experimental-features", "ca-derivations");
auto * v = state.allocValue();
state.mkOutputString(*v, drvPath, outputName.name, std::nullopt);
auto [d, _] = state.coerceToDerivedPathUnchecked(noPos, *v, "");
DerivedPath::Built b {
.drvPath = drvPath,
.outputs = OutputsSpec::Names { outputName.name },
state.mkOutputString(*v, drvPath, outputName.name, std::nullopt, mockXpSettings);
auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, "");
SingleDerivedPath::Built b {
.drvPath = makeConstantStorePathRef(drvPath),
.output = outputName.name,
};
RC_ASSERT(DerivedPath { b } == d);
RC_ASSERT(SingleDerivedPath { b } == d);
}
RC_GTEST_FIXTURE_PROP(
@ -54,12 +61,12 @@ RC_GTEST_FIXTURE_PROP(
{
auto * v = state.allocValue();
state.mkOutputString(*v, drvPath, outputName.name, outPath);
auto [d, _] = state.coerceToDerivedPathUnchecked(noPos, *v, "");
DerivedPath::Built b {
.drvPath = drvPath,
.outputs = OutputsSpec::Names { outputName.name },
auto [d, _] = state.coerceToSingleDerivedPathUnchecked(noPos, *v, "");
SingleDerivedPath::Built b {
.drvPath = makeConstantStorePathRef(drvPath),
.output = outputName.name,
};
RC_ASSERT(DerivedPath { b } == d);
RC_ASSERT(SingleDerivedPath { b } == d);
}
} /* namespace nix */

90
src/libexpr/tests/search-path.cc Normal file
View file

@ -0,0 +1,90 @@
#include <gtest/gtest.h>
#include <gmock/gmock.h>
#include "search-path.hh"
namespace nix {
TEST(SearchPathElem, parse_justPath) {
ASSERT_EQ(
SearchPath::Elem::parse("foo"),
(SearchPath::Elem {
.prefix = SearchPath::Prefix { .s = "" },
.path = SearchPath::Path { .s = "foo" },
}));
}
TEST(SearchPathElem, parse_emptyPrefix) {
ASSERT_EQ(
SearchPath::Elem::parse("=foo"),
(SearchPath::Elem {
.prefix = SearchPath::Prefix { .s = "" },
.path = SearchPath::Path { .s = "foo" },
}));
}
TEST(SearchPathElem, parse_oneEq) {
ASSERT_EQ(
SearchPath::Elem::parse("foo=bar"),
(SearchPath::Elem {
.prefix = SearchPath::Prefix { .s = "foo" },
.path = SearchPath::Path { .s = "bar" },
}));
}
TEST(SearchPathElem, parse_twoEqs) {
ASSERT_EQ(
SearchPath::Elem::parse("foo=bar=baz"),
(SearchPath::Elem {
.prefix = SearchPath::Prefix { .s = "foo" },
.path = SearchPath::Path { .s = "bar=baz" },
}));
}
TEST(SearchPathElem, suffixIfPotentialMatch_justPath) {
SearchPath::Prefix prefix { .s = "" };
ASSERT_EQ(prefix.suffixIfPotentialMatch("any/thing"), std::optional { "any/thing" });
}
TEST(SearchPathElem, suffixIfPotentialMatch_misleadingPrefix1) {
SearchPath::Prefix prefix { .s = "foo" };
ASSERT_EQ(prefix.suffixIfPotentialMatch("fooX"), std::nullopt);
}
TEST(SearchPathElem, suffixIfPotentialMatch_misleadingPrefix2) {
SearchPath::Prefix prefix { .s = "foo" };
ASSERT_EQ(prefix.suffixIfPotentialMatch("fooX/bar"), std::nullopt);
}
TEST(SearchPathElem, suffixIfPotentialMatch_partialPrefix) {
SearchPath::Prefix prefix { .s = "fooX" };
ASSERT_EQ(prefix.suffixIfPotentialMatch("foo"), std::nullopt);
}
TEST(SearchPathElem, suffixIfPotentialMatch_exactPrefix) {
SearchPath::Prefix prefix { .s = "foo" };
ASSERT_EQ(prefix.suffixIfPotentialMatch("foo"), std::optional { "" });
}
TEST(SearchPathElem, suffixIfPotentialMatch_multiKey) {
SearchPath::Prefix prefix { .s = "foo/bar" };
ASSERT_EQ(prefix.suffixIfPotentialMatch("foo/bar/baz"), std::optional { "baz" });
}
TEST(SearchPathElem, suffixIfPotentialMatch_trailingSlash) {
SearchPath::Prefix prefix { .s = "foo" };
ASSERT_EQ(prefix.suffixIfPotentialMatch("foo/"), std::optional { "" });
}
TEST(SearchPathElem, suffixIfPotentialMatch_trailingDoubleSlash) {
SearchPath::Prefix prefix { .s = "foo" };
ASSERT_EQ(prefix.suffixIfPotentialMatch("foo//"), std::optional { "/" });
}
TEST(SearchPathElem, suffixIfPotentialMatch_trailingPath) {
SearchPath::Prefix prefix { .s = "foo" };
ASSERT_EQ(prefix.suffixIfPotentialMatch("foo/bar/baz"), std::optional { "bar/baz" });
}
}

70
src/libexpr/tests/value/context.cc
View file

@ -8,6 +8,8 @@
namespace nix {
// Test a few cases of invalid string context elements.
TEST(NixStringContextElemTest, empty_invalid) {
EXPECT_THROW(
NixStringContextElem::parse(""),
@ -38,6 +40,10 @@ TEST(NixStringContextElemTest, slash_invalid) {
BadStorePath);
}
/**
* Round trip (string <-> data structure) test for
* `NixStringContextElem::Opaque`.
*/
TEST(NixStringContextElemTest, opaque) {
std::string_view opaque = "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x";
auto elem = NixStringContextElem::parse(opaque);
@ -47,6 +53,10 @@ TEST(NixStringContextElemTest, opaque) {
ASSERT_EQ(elem.to_string(), opaque);
}
/**
* Round trip (string <-> data structure) test for
* `NixStringContextElem::DrvDeep`.
*/
TEST(NixStringContextElemTest, drvDeep) {
std::string_view drvDeep = "=g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv";
auto elem = NixStringContextElem::parse(drvDeep);
@ -56,28 +66,62 @@ TEST(NixStringContextElemTest, drvDeep) {
ASSERT_EQ(elem.to_string(), drvDeep);
}
TEST(NixStringContextElemTest, built) {
/**
* Round trip (string <-> data structure) test for a simpler
* `NixStringContextElem::Built`.
*/
TEST(NixStringContextElemTest, built_opaque) {
std::string_view built = "!foo!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv";
auto elem = NixStringContextElem::parse(built);
auto * p = std::get_if<NixStringContextElem::Built>(&elem);
ASSERT_TRUE(p);
ASSERT_EQ(p->output, "foo");
ASSERT_EQ(p->drvPath, StorePath { built.substr(5) });
ASSERT_EQ(*p->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque {
.path = StorePath { built.substr(5) },
}));
ASSERT_EQ(elem.to_string(), built);
}
/**
* Round trip (string <-> data structure) test for a more complex,
* inductive `NixStringContextElem::Built`.
*/
TEST(NixStringContextElemTest, built_built) {
/**
* We set these in tests rather than the regular globals so we don't have
* to worry about race conditions if the tests run concurrently.
*/
ExperimentalFeatureSettings mockXpSettings;
mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations");
std::string_view built = "!foo!bar!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv";
auto elem = NixStringContextElem::parse(built, mockXpSettings);
auto * p = std::get_if<NixStringContextElem::Built>(&elem);
ASSERT_TRUE(p);
ASSERT_EQ(p->output, "foo");
auto * drvPath = std::get_if<SingleDerivedPath::Built>(&*p->drvPath);
ASSERT_TRUE(drvPath);
ASSERT_EQ(drvPath->output, "bar");
ASSERT_EQ(*drvPath->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque {
.path = StorePath { built.substr(9) },
}));
ASSERT_EQ(elem.to_string(), built);
}
/**
* Without the right experimental features enabled, we cannot parse a
* complex inductive string context element.
*/
TEST(NixStringContextElemTest, built_built_xp) {
ASSERT_THROW(
NixStringContextElem::parse("!foo!bar!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"), MissingExperimentalFeature);
}
}
namespace rc {
using namespace nix;
Gen<NixStringContextElem::Opaque> Arbitrary<NixStringContextElem::Opaque>::arbitrary()
{
return gen::just(NixStringContextElem::Opaque {
.path = *gen::arbitrary<StorePath>(),
});
}
Gen<NixStringContextElem::DrvDeep> Arbitrary<NixStringContextElem::DrvDeep>::arbitrary()
{
return gen::just(NixStringContextElem::DrvDeep {
@ -85,14 +129,6 @@ Gen<NixStringContextElem::DrvDeep> Arbitrary<NixStringContextElem::DrvDeep>::arb
});
}
Gen<NixStringContextElem::Built> Arbitrary<NixStringContextElem::Built>::arbitrary()
{
return gen::just(NixStringContextElem::Built {
.drvPath = *gen::arbitrary<StorePath>(),
.output = (*gen::arbitrary<StorePathName>()).name,
});
}
Gen<NixStringContextElem> Arbitrary<NixStringContextElem>::arbitrary()
{
switch (*gen::inRange<uint8_t>(0, std::variant_size_v<NixStringContextElem::Raw>)) {

18
src/libexpr/value-to-json.cc
View file

@ -43,6 +43,7 @@ json printValueAsJSON(EvalState & state, bool strict,
break;
case nNull:
// already initialized as null
break;
case nAttrs: {
@ -59,7 +60,13 @@ json printValueAsJSON(EvalState & state, bool strict,
names.emplace(state.symbols[j.name]);
for (auto & j : names) {
Attr & a(*v.attrs->find(state.symbols.create(j)));
try {
out[j] = printValueAsJSON(state, strict, *a.value, a.pos, context, copyToStore);
} catch (Error & e) {
e.addTrace(state.positions[a.pos],
hintfmt("while evaluating attribute '%1%'", j));
throw;
}
}
} else
return printValueAsJSON(state, strict, *i->value, i->pos, context, copyToStore);
@ -68,8 +75,17 @@ json printValueAsJSON(EvalState & state, bool strict,
case nList: {
out = json::array();
for (auto elem : v.listItems())
int i = 0;
for (auto elem : v.listItems()) {
try {
out.push_back(printValueAsJSON(state, strict, *elem, pos, context, copyToStore));
} catch (Error & e) {
e.addTrace({},
hintfmt("while evaluating list element at index %1%", i));
throw;
}
i++;
}
break;
}

97
src/libexpr/value/context.cc
View file

@ -4,29 +4,52 @@
namespace nix {
NixStringContextElem NixStringContextElem::parse(std::string_view s0)
NixStringContextElem NixStringContextElem::parse(
std::string_view s0,
const ExperimentalFeatureSettings & xpSettings)
{
std::string_view s = s0;
std::function<SingleDerivedPath()> parseRest;
parseRest = [&]() -> SingleDerivedPath {
// Case on whether there is a '!'
size_t index = s.find("!");
if (index == std::string_view::npos) {
return SingleDerivedPath::Opaque {
.path = StorePath { s },
};
} else {
std::string output { s.substr(0, index) };
// Advance string to parse after the '!'
s = s.substr(index + 1);
auto drv = make_ref<SingleDerivedPath>(parseRest());
drvRequireExperiment(*drv, xpSettings);
return SingleDerivedPath::Built {
.drvPath = std::move(drv),
.output = std::move(output),
};
}
};
if (s.size() == 0) {
throw BadNixStringContextElem(s0,
"String context element should never be an empty string");
}
switch (s.at(0)) {
case '!': {
s = s.substr(1); // advance string to parse after first !
size_t index = s.find("!");
// This makes index + 1 safe. Index can be the length (one after index
// of last character), so given any valid character index --- a
// successful find --- we can add one.
if (index == std::string_view::npos) {
// Advance string to parse after the '!'
s = s.substr(1);
// Find *second* '!'
if (s.find("!") == std::string_view::npos) {
throw BadNixStringContextElem(s0,
"String content element beginning with '!' should have a second '!'");
}
return NixStringContextElem::Built {
.drvPath = StorePath { s.substr(index + 1) },
.output = std::string(s.substr(0, index)),
};
return std::visit(
[&](auto x) -> NixStringContextElem { return std::move(x); },
parseRest());
}
case '=': {
return NixStringContextElem::DrvDeep {
@ -34,33 +57,51 @@ NixStringContextElem NixStringContextElem::parse(std::string_view s0)
};
}
default: {
return NixStringContextElem::Opaque {
.path = StorePath { s },
};
// Ensure no '!'
if (s.find("!") != std::string_view::npos) {
throw BadNixStringContextElem(s0,
"String content element not beginning with '!' should not have a second '!'");
}
return std::visit(
[&](auto x) -> NixStringContextElem { return std::move(x); },
parseRest());
}
}
}
std::string NixStringContextElem::to_string() const {
return std::visit(overloaded {
[&](const NixStringContextElem::Built & b) {
std::string NixStringContextElem::to_string() const
{
std::string res;
res += '!';
res += b.output;
res += '!';
res += b.drvPath.to_string();
return res;
std::function<void(const SingleDerivedPath &)> toStringRest;
toStringRest = [&](auto & p) {
std::visit(overloaded {
[&](const SingleDerivedPath::Opaque & o) {
res += o.path.to_string();
},
[&](const NixStringContextElem::DrvDeep & d) {
std::string res;
res += '=';
res += d.drvPath.to_string();
return res;
[&](const SingleDerivedPath::Built & o) {
res += o.output;
res += '!';
toStringRest(*o.drvPath);
},
}, p.raw());
};
std::visit(overloaded {
[&](const NixStringContextElem::Built & b) {
res += '!';
toStringRest(b);
},
[&](const NixStringContextElem::Opaque & o) {
return std::string { o.path.to_string() };
toStringRest(o);
},
[&](const NixStringContextElem::DrvDeep & d) {
res += '=';
res += d.drvPath.to_string();
},
}, raw());
return res;
}
}

21
src/libexpr/value/context.hh
View file

@ -3,7 +3,7 @@
#include "util.hh"
#include "comparator.hh"
#include "path.hh"
#include "derived-path.hh"
#include <variant>
@ -31,11 +31,7 @@ public:
*
* Encoded as just the path: ‘<path>’.
*/
struct NixStringContextElem_Opaque {
StorePath path;
GENERATE_CMP(NixStringContextElem_Opaque, me->path);
};
typedef SingleDerivedPath::Opaque NixStringContextElem_Opaque;
/**
* Path to a derivation and its entire build closure.
@ -57,12 +53,7 @@ struct NixStringContextElem_DrvDeep {
*
* Encoded in the form ‘!<output>!<drvPath>’.
*/
struct NixStringContextElem_Built {
StorePath drvPath;
std::string output;
GENERATE_CMP(NixStringContextElem_Built, me->drvPath, me->output);
};
typedef SingleDerivedPath::Built NixStringContextElem_Built;
using _NixStringContextElem_Raw = std::variant<
NixStringContextElem_Opaque,
@ -93,8 +84,12 @@ struct NixStringContextElem : _NixStringContextElem_Raw {
* - ‘<path>’
* - ‘=<path>’
* - ‘!<name>!<path>’
*
* @param xpSettings Stop-gap to avoid globals during unit tests.
*/
static NixStringContextElem parse(std::string_view s);
static NixStringContextElem parse(
std::string_view s,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
std::string to_string() const;
};

10
src/libfetchers/fetchers.cc
View file

@ -13,9 +13,9 @@ void registerInputScheme(std::shared_ptr<InputScheme> && inputScheme)
inputSchemes->push_back(std::move(inputScheme));
}
Input Input::fromURL(const std::string & url)
Input Input::fromURL(const std::string & url, bool requireTree)
{
return fromURL(parseURL(url));
return fromURL(parseURL(url), requireTree);
}
static void fixupInput(Input & input)
@ -31,10 +31,10 @@ static void fixupInput(Input & input)
input.locked = true;
}
Input Input::fromURL(const ParsedURL & url)
Input Input::fromURL(const ParsedURL & url, bool requireTree)
{
for (auto & inputScheme : *inputSchemes) {
auto res = inputScheme->inputFromURL(url);
auto res = inputScheme->inputFromURL(url, requireTree);
if (res) {
res->scheme = inputScheme;
fixupInput(*res);
@ -217,10 +217,8 @@ StorePath Input::computeStorePath(Store & store) const
if (!narHash)
throw Error("cannot compute store path for unlocked input '%s'", to_string());
return store.makeFixedOutputPath(getName(), FixedOutputInfo {
.hash = {
.method = FileIngestionMethod::Recursive,
.hash = *narHash,
},
.references = {},
});
}

6
src/libfetchers/fetchers.hh
View file

@ -44,9 +44,9 @@ struct Input
std::optional<Path> parent;
public:
static Input fromURL(const std::string & url);
static Input fromURL(const std::string & url, bool requireTree = true);
static Input fromURL(const ParsedURL & url);
static Input fromURL(const ParsedURL & url, bool requireTree = true);
static Input fromAttrs(Attrs && attrs);
@ -129,7 +129,7 @@ struct InputScheme
virtual ~InputScheme()
{ }
virtual std::optional<Input> inputFromURL(const ParsedURL & url) const = 0;
virtual std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const = 0;
virtual std::optional<Input> inputFromAttrs(const Attrs & attrs) const = 0;

2
src/libfetchers/git.cc
View file

@ -256,7 +256,7 @@ std::pair<StorePath, Input> fetchFromWorkdir(ref<Store> store, Input & input, co
struct GitInputScheme : InputScheme
{
std::optional<Input> inputFromURL(const ParsedURL & url) const override
std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const override
{
if (url.scheme != "git" &&
url.scheme != "git+http" &&

2
src/libfetchers/github.cc
View file

@ -30,7 +30,7 @@ struct GitArchiveInputScheme : InputScheme
virtual std::optional<std::pair<std::string, std::string>> accessHeaderFromToken(const std::string & token) const = 0;
std::optional<Input> inputFromURL(const ParsedURL & url) const override
std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const override
{
if (url.scheme != type()) return {};

2
src/libfetchers/indirect.cc
View file

@ -7,7 +7,7 @@ std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
struct IndirectInputScheme : InputScheme
{
std::optional<Input> inputFromURL(const ParsedURL & url) const override
std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const override
{
if (url.scheme != "flake") return {};

2
src/libfetchers/mercurial.cc
View file

@ -43,7 +43,7 @@ static std::string runHg(const Strings & args, const std::optional<std::string>
struct MercurialInputScheme : InputScheme
{
std::optional<Input> inputFromURL(const ParsedURL & url) const override
std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const override
{
if (url.scheme != "hg+http" &&
url.scheme != "hg+https" &&

2
src/libfetchers/path.cc
View file

@ -6,7 +6,7 @@ namespace nix::fetchers {
struct PathInputScheme : InputScheme
{
std::optional<Input> inputFromURL(const ParsedURL & url) const override
std::optional<Input> inputFromURL(const ParsedURL & url, bool requireTree) const override
{
if (url.scheme != "path") return {};

16
src/libfetchers/tarball.cc
View file

@ -77,10 +77,8 @@ DownloadFileResult downloadFile(
*store,
name,
FixedOutputInfo {
.hash = {
.method = FileIngestionMethod::Flat,
.hash = hash,
},
.references = {},
},
hashString(htSHA256, sink.s),
@ -196,11 +194,11 @@ struct CurlInputScheme : InputScheme
|| hasSuffix(path, ".tar.zst");
}
virtual bool isValidURL(const ParsedURL & url) const = 0;
virtual bool isValidURL(const ParsedURL & url, bool requireTree) const = 0;
std::optional<Input> inputFromURL(const ParsedURL & _url) const override
std::optional<Input> inputFromURL(const ParsedURL & _url, bool requireTree) const override
{
if (!isValidURL(_url))
if (!isValidURL(_url, requireTree))
return std::nullopt;
Input input;
@ -267,13 +265,13 @@ struct FileInputScheme : CurlInputScheme
{
const std::string inputType() const override { return "file"; }
bool isValidURL(const ParsedURL & url) const override
bool isValidURL(const ParsedURL & url, bool requireTree) const override
{
auto parsedUrlScheme = parseUrlScheme(url.scheme);
return transportUrlSchemes.count(std::string(parsedUrlScheme.transport))
&& (parsedUrlScheme.application
? parsedUrlScheme.application.value() == inputType()
: !hasTarballExtension(url.path));
: (!requireTree && !hasTarballExtension(url.path)));
}
std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input) override
@ -287,14 +285,14 @@ struct TarballInputScheme : CurlInputScheme
{
const std::string inputType() const override { return "tarball"; }
bool isValidURL(const ParsedURL & url) const override
bool isValidURL(const ParsedURL & url, bool requireTree) const override
{
auto parsedUrlScheme = parseUrlScheme(url.scheme);
return transportUrlSchemes.count(std::string(parsedUrlScheme.transport))
&& (parsedUrlScheme.application
? parsedUrlScheme.application.value() == inputType()
: hasTarballExtension(url.path));
: (requireTree || hasTarballExtension(url.path)));
}
std::pair<StorePath, Input> fetch(ref<Store> store, const Input & _input) override

8
src/libstore/binary-cache-store.cc
View file

@ -309,10 +309,8 @@ StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view n
*this,
name,
FixedOutputInfo {
.hash = {
.method = method,
.hash = nar.first,
},
.references = {
.others = references,
// caller is not capable of creating a self-reference, because this is content-addressed without modulus
@ -428,10 +426,8 @@ StorePath BinaryCacheStore::addToStore(
*this,
name,
FixedOutputInfo {
.hash = {
.method = method,
.hash = h,
},
.references = {
.others = references,
// caller is not capable of creating a self-reference, because this is content-addressed without modulus
@ -465,8 +461,8 @@ StorePath BinaryCacheStore::addTextToStore(
*this,
std::string { name },
TextInfo {
{ .hash = textHash },
references,
.hash = textHash,
.references = references,
},
nar.first,
};

10
src/libstore/build/derivation-goal.cc
View file

@ -65,7 +65,7 @@ namespace nix {
DerivationGoal::DerivationGoal(const StorePath & drvPath,
const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode)
: Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs })
: Goal(worker, DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = wantedOutputs })
, useDerivation(true)
, drvPath(drvPath)
, wantedOutputs(wantedOutputs)
@ -74,7 +74,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath,
state = &DerivationGoal::getDerivation;
name = fmt(
"building of '%s' from .drv file",
DerivedPath::Built { drvPath, wantedOutputs }.to_string(worker.store));
DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store));
trace("created");
mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
@ -84,7 +84,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath,
DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode)
: Goal(worker, DerivedPath::Built { .drvPath = drvPath, .outputs = wantedOutputs })
: Goal(worker, DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = wantedOutputs })
, useDerivation(false)
, drvPath(drvPath)
, wantedOutputs(wantedOutputs)
@ -95,7 +95,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation
state = &DerivationGoal::haveDerivation;
name = fmt(
"building of '%s' from in-memory derivation",
DerivedPath::Built { drvPath, drv.outputNames() }.to_string(worker.store));
DerivedPath::Built { makeConstantStorePathRef(drvPath), drv.outputNames() }.to_string(worker.store));
trace("created");
mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
@ -1490,7 +1490,7 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result)
for (auto & outputName : outputs->second) {
auto buildResult = dg->getBuildResult(DerivedPath::Built {
.drvPath = dg->drvPath,
.drvPath = makeConstantStorePathRef(dg->drvPath),
.outputs = OutputsSpec::Names { outputName },
});
if (buildResult.success()) {

2
src/libstore/build/entry-points.cc
View file

@ -77,7 +77,7 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat
try {
worker.run(Goals{goal});
return goal->getBuildResult(DerivedPath::Built {
.drvPath = drvPath,
.drvPath = makeConstantStorePathRef(drvPath),
.outputs = OutputsSpec::All {},
});
} catch (Error & e) {

50
src/libstore/build/local-derivation-goal.cc
View file

@ -1,5 +1,5 @@
#include "local-derivation-goal.hh"
#include "gc-store.hh"
#include "indirect-root-store.hh"
#include "hook-instance.hh"
#include "worker.hh"
#include "builtins.hh"
@ -594,6 +594,10 @@ void LocalDerivationGoal::startBuilder()
else
dirsInChroot[i.substr(0, p)] = {i.substr(p + 1), optional};
}
if (hasPrefix(worker.store.storeDir, tmpDirInSandbox))
{
throw Error("`sandbox-build-dir` must not contain the storeDir");
}
dirsInChroot[tmpDirInSandbox] = tmpDir;
/* Add the closure of store paths to the chroot. */
@ -908,15 +912,13 @@ void LocalDerivationGoal::startBuilder()
openSlave();
/* Drop additional groups here because we can't do it
after we've created the new user namespace. FIXME:
this means that if we're not root in the parent
namespace, we can't drop additional groups; they will
be mapped to nogroup in the child namespace. There does
not seem to be a workaround for this. (But who can tell
from reading user_namespaces(7)?)
See also https://lwn.net/Articles/621612/. */
if (getuid() == 0 && setgroups(0, 0) == -1)
after we've created the new user namespace. */
if (setgroups(0, 0) == -1) {
if (errno != EPERM)
throw SysError("setgroups failed");
if (settings.requireDropSupplementaryGroups)
throw Error("setgroups failed. Set the require-drop-supplementary-groups option to false to skip this step.");
}
ProcessOptions options;
options.cloneFlags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD;
@ -1170,6 +1172,19 @@ void LocalDerivationGoal::writeStructuredAttrs()
}
static StorePath pathPartOfReq(const SingleDerivedPath & req)
{
return std::visit(overloaded {
[&](const SingleDerivedPath::Opaque & bo) {
return bo.path;
},
[&](const SingleDerivedPath::Built & bfd) {
return pathPartOfReq(*bfd.drvPath);
},
}, req.raw());
}
static StorePath pathPartOfReq(const DerivedPath & req)
{
return std::visit(overloaded {
@ -1177,7 +1192,7 @@ static StorePath pathPartOfReq(const DerivedPath & req)
return bo.path;
},
[&](const DerivedPath::Built & bfd) {
return bfd.drvPath;
return pathPartOfReq(*bfd.drvPath);
},
}, req.raw());
}
@ -1198,7 +1213,7 @@ struct RestrictedStoreConfig : virtual LocalFSStoreConfig
/* A wrapper around LocalStore that only allows building/querying of
paths that are in the input closures of the build or were added via
recursive Nix calls. */
struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore, public virtual GcStore
struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual IndirectRootStore, public virtual GcStore
{
ref<LocalStore> next;
@ -1249,11 +1264,13 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
void queryReferrers(const StorePath & path, StorePathSet & referrers) override
{ }
std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override
std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(
const StorePath & path,
Store * evalStore = nullptr) override
{
if (!goal.isAllowed(path))
throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path));
return next->queryPartialDerivationOutputMap(path);
return next->queryPartialDerivationOutputMap(path, evalStore);
}
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
@ -2303,7 +2320,6 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
bool discardReferences = false;
if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) {
if (auto udr = get(*structuredAttrs, "unsafeDiscardReferences")) {
experimentalFeatureSettings.require(Xp::DiscardReferences);
if (auto output = get(*udr, outputName)) {
if (!output->is_boolean())
throw Error("attribute 'unsafeDiscardReferences.\"%s\"' of derivation '%s' must be a Boolean", outputName, drvPath.to_string());
@ -2538,16 +2554,16 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
},
[&](const DerivationOutput::CAFixed & dof) {
auto wanted = dof.ca.getHash();
auto & wanted = dof.ca.hash;
auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating {
.method = dof.ca.getMethod(),
.method = dof.ca.method,
.hashType = wanted.type,
});
/* Check wanted hash */
assert(newInfo0.ca);
auto got = newInfo0.ca->getHash();
auto & got = newInfo0.ca->hash;
if (wanted != got) {
/* Throw an error after registering the path as
valid. */

7
src/libstore/build/worker.cc
View file

@ -111,7 +111,10 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode)
{
return std::visit(overloaded {
[&](const DerivedPath::Built & bfd) -> GoalPtr {
return makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode);
if (auto bop = std::get_if<DerivedPath::Opaque>(&*bfd.drvPath))
return makeDerivationGoal(bop->path, bfd.outputs, buildMode);
else
throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented.");
},
[&](const DerivedPath::Opaque & bo) -> GoalPtr {
return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair);
@ -265,7 +268,7 @@ void Worker::run(const Goals & _topGoals)
for (auto & i : _topGoals) {
topGoals.insert(i);
if (auto goal = dynamic_cast<DerivationGoal *>(i.get())) {
topPaths.push_back(DerivedPath::Built{goal->drvPath, goal->wantedOutputs});
topPaths.push_back(DerivedPath::Built{makeConstantStorePathRef(goal->drvPath), goal->wantedOutputs});
} else if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) {
topPaths.push_back(DerivedPath::Opaque{goal->storePath});
}

119
src/libstore/content-address.cc
View file

@ -4,11 +4,6 @@
namespace nix {
std::string FixedOutputHash::printMethodAlgo() const
{
return makeFileIngestionPrefix(method) + printHashType(hash.type);
}
std::string makeFileIngestionPrefix(FileIngestionMethod m)
{
switch (m) {
@ -42,21 +37,6 @@ ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m)
return method;
}
std::string ContentAddress::render() const
{
return std::visit(overloaded {
[](const TextHash & th) {
return "text:"
+ th.hash.to_string(Base32, true);
},
[](const FixedOutputHash & fsh) {
return "fixed:"
+ makeFileIngestionPrefix(fsh.method)
+ fsh.hash.to_string(Base32, true);
}
}, raw);
}
std::string ContentAddressMethod::render(HashType ht) const
{
return std::visit(overloaded {
@ -69,6 +49,20 @@ std::string ContentAddressMethod::render(HashType ht) const
}, raw);
}
std::string ContentAddress::render() const
{
return std::visit(overloaded {
[](const TextIngestionMethod &) -> std::string {
return "text:";
},
[](const FileIngestionMethod & method) {
return "fixed:"
+ makeFileIngestionPrefix(method);
},
}, method.raw)
+ this->hash.to_string(Base32, true);
}
/**
* Parses content address strings up to the hash.
*/
@ -118,22 +112,12 @@ ContentAddress ContentAddress::parse(std::string_view rawCa)
{
auto rest = rawCa;
auto [caMethod, hashType_] = parseContentAddressMethodPrefix(rest);
auto hashType = hashType_; // work around clang bug
auto [caMethod, hashType] = parseContentAddressMethodPrefix(rest);
return std::visit(overloaded {
[&](TextIngestionMethod &) {
return ContentAddress(TextHash {
.hash = Hash::parseNonSRIUnprefixed(rest, hashType)
});
},
[&](FileIngestionMethod & fim) {
return ContentAddress(FixedOutputHash {
.method = fim,
return ContentAddress {
.method = std::move(caMethod).raw,
.hash = Hash::parseNonSRIUnprefixed(rest, hashType),
});
},
}, caMethod.raw);
};
}
std::pair<ContentAddressMethod, HashType> ContentAddressMethod::parse(std::string_view caMethod)
@ -156,52 +140,10 @@ std::string renderContentAddress(std::optional<ContentAddress> ca)
return ca ? ca->render() : "";
}
ContentAddress ContentAddress::fromParts(
ContentAddressMethod method, Hash hash) noexcept
{
return std::visit(overloaded {
[&](TextIngestionMethod _) -> ContentAddress {
return TextHash {
.hash = std::move(hash),
};
},
[&](FileIngestionMethod m2) -> ContentAddress {
return FixedOutputHash {
.method = std::move(m2),
.hash = std::move(hash),
};
},
}, method.raw);
}
ContentAddressMethod ContentAddress::getMethod() const
{
return std::visit(overloaded {
[](const TextHash & th) -> ContentAddressMethod {
return TextIngestionMethod {};
},
[](const FixedOutputHash & fsh) -> ContentAddressMethod {
return fsh.method;
},
}, raw);
}
const Hash & ContentAddress::getHash() const
{
return std::visit(overloaded {
[](const TextHash & th) -> auto & {
return th.hash;
},
[](const FixedOutputHash & fsh) -> auto & {
return fsh.hash;
},
}, raw);
}
std::string ContentAddress::printMethodAlgo() const
{
return getMethod().renderPrefix()
+ printHashType(getHash().type);
return method.renderPrefix()
+ printHashType(hash.type);
}
bool StoreReferences::empty() const
@ -217,19 +159,20 @@ size_t StoreReferences::size() const
ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const ContentAddress & ca) noexcept
{
return std::visit(overloaded {
[&](const TextHash & h) -> ContentAddressWithReferences {
[&](const TextIngestionMethod &) -> ContentAddressWithReferences {
return TextInfo {
.hash = h,
.hash = ca.hash,
.references = {},
};
},
[&](const FixedOutputHash & h) -> ContentAddressWithReferences {
[&](const FileIngestionMethod & method) -> ContentAddressWithReferences {
return FixedOutputInfo {
.hash = h,
.method = method,
.hash = ca.hash,
.references = {},
};
},
}, ca.raw);
}, ca.method.raw);
}
std::optional<ContentAddressWithReferences> ContentAddressWithReferences::fromPartsOpt(
@ -241,7 +184,7 @@ std::optional<ContentAddressWithReferences> ContentAddressWithReferences::fromPa
return std::nullopt;
return ContentAddressWithReferences {
TextInfo {
.hash = { .hash = std::move(hash) },
.hash = std::move(hash),
.references = std::move(refs.others),
}
};
@ -249,10 +192,8 @@ std::optional<ContentAddressWithReferences> ContentAddressWithReferences::fromPa
[&](FileIngestionMethod m2) -> std::optional<ContentAddressWithReferences> {
return ContentAddressWithReferences {
FixedOutputInfo {
.hash = {
.method = m2,
.hash = std::move(hash),
},
.references = std::move(refs),
}
};
@ -267,7 +208,7 @@ ContentAddressMethod ContentAddressWithReferences::getMethod() const
return TextIngestionMethod {};
},
[](const FixedOutputInfo & fsh) -> ContentAddressMethod {
return fsh.hash.method;
return fsh.method;
},
}, raw);
}
@ -276,10 +217,10 @@ Hash ContentAddressWithReferences::getHash() const
{
return std::visit(overloaded {
[](const TextInfo & th) {
return th.hash.hash;
return th.hash;
},
[](const FixedOutputInfo & fsh) {
return fsh.hash.hash;
return fsh.hash;
},
}, raw);
}

91
src/libstore/content-address.hh
View file

@ -113,37 +113,6 @@ struct ContentAddressMethod
* Mini content address
*/
/**
* Somewhat obscure, used by \ref Derivation derivations and
* `builtins.toFile` currently.
*/
struct TextHash {
/**
* Hash of the contents of the text/file.
*/
Hash hash;
GENERATE_CMP(TextHash, me->hash);
};
/**
* Used by most store objects that are content-addressed.
*/
struct FixedOutputHash {
/**
* How the file system objects are serialized
*/
FileIngestionMethod method;
/**
* Hash of that serialization
*/
Hash hash;
std::string printMethodAlgo() const;
GENERATE_CMP(FixedOutputHash, me->method, me->hash);
};
/**
* We've accumulated several types of content-addressed paths over the
* years; fixed-output derivations support multiple hash algorithms and
@ -158,19 +127,17 @@ struct FixedOutputHash {
*/
struct ContentAddress
{
typedef std::variant<
TextHash,
FixedOutputHash
> Raw;
/**
* How the file system objects are serialized
*/
ContentAddressMethod method;
Raw raw;
/**
* Hash of that serialization
*/
Hash hash;
GENERATE_CMP(ContentAddress, me->raw);
/* The moral equivalent of `using Raw::Raw;` */
ContentAddress(auto &&... arg)
: raw(std::forward<decltype(arg)>(arg)...)
{ }
GENERATE_CMP(ContentAddress, me->method, me->hash);
/**
* Compute the content-addressability assertion
@ -183,20 +150,6 @@ struct ContentAddress
static std::optional<ContentAddress> parseOpt(std::string_view rawCaOpt);
/**
* Create a `ContentAddress` from 2 parts:
*
* @param method Way ingesting the file system data.
*
* @param hash Hash of ingested file system data.
*/
static ContentAddress fromParts(
ContentAddressMethod method, Hash hash) noexcept;
ContentAddressMethod getMethod() const;
const Hash & getHash() const;
std::string printMethodAlgo() const;
};
@ -219,7 +172,8 @@ std::string renderContentAddress(std::optional<ContentAddress> ca);
* References to other store objects are tracked with store paths, self
* references however are tracked with a boolean.
*/
struct StoreReferences {
struct StoreReferences
{
/**
* References to other store objects
*/
@ -246,8 +200,13 @@ struct StoreReferences {
};
// This matches the additional info that we need for makeTextPath
struct TextInfo {
TextHash hash;
struct TextInfo
{
/**
* Hash of the contents of the text/file.
*/
Hash hash;
/**
* References to other store objects only; self references
* disallowed
@ -257,8 +216,18 @@ struct TextInfo {
GENERATE_CMP(TextInfo, me->hash, me->references);
};
struct FixedOutputInfo {
FixedOutputHash hash;
struct FixedOutputInfo
{
/**
* How the file system objects are serialized
*/
FileIngestionMethod method;
/**
* Hash of that serialization
*/
Hash hash;
/**
* References to other store objects or this one.
*/

5
src/libstore/daemon.cc
View file

@ -7,6 +7,7 @@
#include "store-cast.hh"
#include "gc-store.hh"
#include "log-store.hh"
#include "indirect-root-store.hh"
#include "path-with-outputs.hh"
#include "finally.hh"
#include "archive.hh"
@ -675,8 +676,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
Path path = absPath(readString(from));
logger->startWork();
auto & gcStore = require<GcStore>(*store);
gcStore.addIndirectRoot(path);
auto & indirectRootStore = require<IndirectRootStore>(*store);
indirectRootStore.addIndirectRoot(path);
logger->stopWork();
to << 1;

64
src/libstore/derivations.cc
View file

@ -232,9 +232,10 @@ static DerivationOutput parseDerivationOutput(const Store & store,
validatePath(pathS);
auto hash = Hash::parseNonSRIUnprefixed(hashS, hashType);
return DerivationOutput::CAFixed {
.ca = ContentAddress::fromParts(
std::move(method),
std::move(hash)),
.ca = ContentAddress {
.method = std::move(method),
.hash = std::move(hash),
},
};
} else {
experimentalFeatureSettings.require(Xp::CaDerivations);
@ -395,7 +396,7 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs,
[&](const DerivationOutput::CAFixed & dof) {
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first)));
s += ','; printUnquotedString(s, dof.ca.printMethodAlgo());
s += ','; printUnquotedString(s, dof.ca.getHash().to_string(Base16, false));
s += ','; printUnquotedString(s, dof.ca.hash.to_string(Base16, false));
},
[&](const DerivationOutput::CAFloating & dof) {
s += ','; printUnquotedString(s, "");
@ -628,7 +629,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
auto & dof = std::get<DerivationOutput::CAFixed>(i.second.raw());
auto hash = hashString(htSHA256, "fixed:out:"
+ dof.ca.printMethodAlgo() + ":"
+ dof.ca.getHash().to_string(Base16, false) + ":"
+ dof.ca.hash.to_string(Base16, false) + ":"
+ store.printStorePath(dof.path(store, drv.name, i.first)));
outputHashes.insert_or_assign(i.first, std::move(hash));
}
@ -780,7 +781,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
[&](const DerivationOutput::CAFixed & dof) {
out << store.printStorePath(dof.path(store, drv.name, i.first))
<< dof.ca.printMethodAlgo()
<< dof.ca.getHash().to_string(Base16, false);
<< dof.ca.hash.to_string(Base16, false);
},
[&](const DerivationOutput::CAFloating & dof) {
out << ""
@ -878,9 +879,11 @@ std::optional<BasicDerivation> Derivation::tryResolve(
for (auto & [inputDrv, inputOutputs] : inputDrvs) {
for (auto & outputName : inputOutputs) {
if (auto actualPath = get(inputDrvOutputs, { inputDrv, outputName })) {
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
inputRewrites.emplace(
DownstreamPlaceholder::unknownCaOutput(inputDrv, outputName).render(),
store.printStorePath(*actualPath));
}
resolved.inputSrcs.insert(*actualPath);
} else {
warn("output '%s' of input '%s' missing, aborting the resolving",
@ -970,7 +973,7 @@ nlohmann::json DerivationOutput::toJSON(
[&](const DerivationOutput::CAFixed & dof) {
res["path"] = store.printStorePath(dof.path(store, drvName, outputName));
res["hashAlgo"] = dof.ca.printMethodAlgo();
res["hash"] = dof.ca.getHash().to_string(Base16, false);
res["hash"] = dof.ca.hash.to_string(Base16, false);
// FIXME print refs?
},
[&](const DerivationOutput::CAFloating & dof) {
@ -992,6 +995,7 @@ DerivationOutput DerivationOutput::fromJSON(
const ExperimentalFeatureSettings & xpSettings)
{
std::set<std::string_view> keys;
ensureType(_json, nlohmann::detail::value_t::object);
auto json = (std::map<std::string, nlohmann::json>) _json;
for (const auto & [key, _] : json)
@ -1017,9 +1021,10 @@ DerivationOutput DerivationOutput::fromJSON(
else if (keys == (std::set<std::string_view> { "path", "hashAlgo", "hash" })) {
auto [method, hashType] = methodAlgo();
auto dof = DerivationOutput::CAFixed {
.ca = ContentAddress::fromParts(
std::move(method),
Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType)),
.ca = ContentAddress {
.method = std::move(method),
.hash = Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType),
},
};
if (dof.path(store, drvName, outputName) != store.parseStorePath((std::string) json["path"]))
throw Error("Path doesn't match derivation output");
@ -1095,36 +1100,51 @@ Derivation Derivation::fromJSON(
const Store & store,
const nlohmann::json & json)
{
using nlohmann::detail::value_t;
Derivation res;
res.name = json["name"];
ensureType(json, value_t::object);
{
auto & outputsObj = json["outputs"];
res.name = ensureType(valueAt(json, "name"), value_t::string);
try {
auto & outputsObj = ensureType(valueAt(json, "outputs"), value_t::object);
for (auto & [outputName, output] : outputsObj.items()) {
res.outputs.insert_or_assign(
outputName,
DerivationOutput::fromJSON(store, res.name, outputName, output));
}
} catch (Error & e) {
e.addTrace({}, "while reading key 'outputs'");
throw;
}
{
auto & inputsList = json["inputSrcs"];
try {
auto & inputsList = ensureType(valueAt(json, "inputSrcs"), value_t::array);
for (auto & input : inputsList)
res.inputSrcs.insert(store.parseStorePath(static_cast<const std::string &>(input)));
} catch (Error & e) {
e.addTrace({}, "while reading key 'inputSrcs'");
throw;
}
{
auto & inputDrvsObj = json["inputDrvs"];
for (auto & [inputDrvPath, inputOutputs] : inputDrvsObj.items())
try {
auto & inputDrvsObj = ensureType(valueAt(json, "inputDrvs"), value_t::object);
for (auto & [inputDrvPath, inputOutputs] : inputDrvsObj.items()) {
ensureType(inputOutputs, value_t::array);
res.inputDrvs[store.parseStorePath(inputDrvPath)] =
static_cast<const StringSet &>(inputOutputs);
}
} catch (Error & e) {
e.addTrace({}, "while reading key 'inputDrvs'");
throw;
}
res.platform = json["system"];
res.builder = json["builder"];
res.args = json["args"];
res.env = json["env"];
res.platform = ensureType(valueAt(json, "system"), value_t::string);
res.builder = ensureType(valueAt(json, "builder"), value_t::string);
res.args = ensureType(valueAt(json, "args"), value_t::array);
res.env = ensureType(valueAt(json, "env"), value_t::object);
return res;
}

285
src/libstore/derived-path.cc
View file

@ -1,5 +1,4 @@
#include "derived-path.hh"
#include "derivations.hh"
#include "store-api.hh"
#include <nlohmann/json.hpp>
@ -8,50 +7,83 @@
namespace nix {
nlohmann::json DerivedPath::Opaque::toJSON(ref<Store> store) const {
#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \
bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \
{ \
const MY_TYPE* me = this; \
auto fields1 = std::make_tuple<const CHILD_TYPE &, const FIELD_TYPE &>(*me->drvPath, me->FIELD); \
me = &other; \
auto fields2 = std::make_tuple<const CHILD_TYPE &, const FIELD_TYPE &>(*me->drvPath, me->FIELD); \
return fields1 COMPARATOR fields2; \
}
#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \
CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \
CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \
CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <)
#define FIELD_TYPE std::string
CMP(SingleDerivedPath, SingleDerivedPathBuilt, output)
#undef FIELD_TYPE
#define FIELD_TYPE OutputsSpec
CMP(SingleDerivedPath, DerivedPathBuilt, outputs)
#undef FIELD_TYPE
#undef CMP
#undef CMP_ONE
nlohmann::json DerivedPath::Opaque::toJSON(const Store & store) const
{
return store.printStorePath(path);
}
nlohmann::json SingleDerivedPath::Built::toJSON(Store & store) const {
nlohmann::json res;
res["path"] = store->printStorePath(path);
res["drvPath"] = drvPath->toJSON(store);
// Fallback for the input-addressed derivation case: We expect to always be
// able to print the output paths, so let’s do it
// FIXME try-resolve on drvPath
const auto outputMap = store.queryPartialDerivationOutputMap(resolveDerivedPath(store, *drvPath));
res["output"] = output;
auto outputPathIter = outputMap.find(output);
if (outputPathIter == outputMap.end())
res["outputPath"] = nullptr;
else if (std::optional p = outputPathIter->second)
res["outputPath"] = store.printStorePath(*p);
else
res["outputPath"] = nullptr;
return res;
}
nlohmann::json DerivedPath::Built::toJSON(ref<Store> store) const {
nlohmann::json DerivedPath::Built::toJSON(Store & store) const {
nlohmann::json res;
res["drvPath"] = store->printStorePath(drvPath);
res["drvPath"] = drvPath->toJSON(store);
// Fallback for the input-addressed derivation case: We expect to always be
// able to print the output paths, so let’s do it
const auto outputMap = store->queryPartialDerivationOutputMap(drvPath);
// FIXME try-resolve on drvPath
const auto outputMap = store.queryPartialDerivationOutputMap(resolveDerivedPath(store, *drvPath));
for (const auto & [output, outputPathOpt] : outputMap) {
if (!outputs.contains(output)) continue;
if (outputPathOpt)
res["outputs"][output] = store->printStorePath(*outputPathOpt);
res["outputs"][output] = store.printStorePath(*outputPathOpt);
else
res["outputs"][output] = nullptr;
}
return res;
}
nlohmann::json BuiltPath::Built::toJSON(ref<Store> store) const {
nlohmann::json res;
res["drvPath"] = store->printStorePath(drvPath);
for (const auto& [output, path] : outputs) {
res["outputs"][output] = store->printStorePath(path);
}
return res;
nlohmann::json SingleDerivedPath::toJSON(Store & store) const
{
return std::visit([&](const auto & buildable) {
return buildable.toJSON(store);
}, raw());
}
StorePathSet BuiltPath::outPaths() const
nlohmann::json DerivedPath::toJSON(Store & store) const
{
return std::visit(
overloaded{
[](const BuiltPath::Opaque & p) { return StorePathSet{p.path}; },
[](const BuiltPath::Built & b) {
StorePathSet res;
for (auto & [_, path] : b.outputs)
res.insert(path);
return res;
},
}, raw()
);
return std::visit([&](const auto & buildable) {
return buildable.toJSON(store);
}, raw());
}
std::string DerivedPath::Opaque::to_string(const Store & store) const
@ -59,25 +91,49 @@ std::string DerivedPath::Opaque::to_string(const Store & store) const
return store.printStorePath(path);
}
std::string SingleDerivedPath::Built::to_string(const Store & store) const
{
return drvPath->to_string(store) + "^" + output;
}
std::string SingleDerivedPath::Built::to_string_legacy(const Store & store) const
{
return drvPath->to_string(store) + "!" + output;
}
std::string DerivedPath::Built::to_string(const Store & store) const
{
return store.printStorePath(drvPath)
return drvPath->to_string(store)
+ '^'
+ outputs.to_string();
}
std::string DerivedPath::Built::to_string_legacy(const Store & store) const
{
return store.printStorePath(drvPath)
+ '!'
return drvPath->to_string_legacy(store)
+ "!"
+ outputs.to_string();
}
std::string SingleDerivedPath::to_string(const Store & store) const
{
return std::visit(
[&](const auto & req) { return req.to_string(store); },
raw());
}
std::string DerivedPath::to_string(const Store & store) const
{
return std::visit(
[&](const auto & req) { return req.to_string(store); },
raw());
}
std::string SingleDerivedPath::to_string_legacy(const Store & store) const
{
return std::visit(overloaded {
[&](const DerivedPath::Built & req) { return req.to_string(store); },
[&](const DerivedPath::Opaque & req) { return req.to_string(store); },
[&](const SingleDerivedPath::Built & req) { return req.to_string_legacy(store); },
[&](const SingleDerivedPath::Opaque & req) { return req.to_string(store); },
}, this->raw());
}
@ -95,61 +151,156 @@ DerivedPath::Opaque DerivedPath::Opaque::parse(const Store & store, std::string_
return {store.parseStorePath(s)};
}
DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_view drvS, std::string_view outputsS)
void drvRequireExperiment(
const SingleDerivedPath & drv,
const ExperimentalFeatureSettings & xpSettings)
{
std::visit(overloaded {
[&](const SingleDerivedPath::Opaque &) {
// plain drv path; no experimental features required.
},
[&](const SingleDerivedPath::Built &) {
xpSettings.require(Xp::DynamicDerivations);
},
}, drv.raw());
}
SingleDerivedPath::Built SingleDerivedPath::Built::parse(
const Store & store, ref<SingleDerivedPath> drv,
std::string_view output,
const ExperimentalFeatureSettings & xpSettings)
{
drvRequireExperiment(*drv, xpSettings);
return {
.drvPath = store.parseStorePath(drvS),
.drvPath = drv,
.output = std::string { output },
};
}
DerivedPath::Built DerivedPath::Built::parse(
const Store & store, ref<SingleDerivedPath> drv,
std::string_view outputsS,
const ExperimentalFeatureSettings & xpSettings)
{
drvRequireExperiment(*drv, xpSettings);
return {
.drvPath = drv,
.outputs = OutputsSpec::parse(outputsS),
};
}
static inline DerivedPath parseWith(const Store & store, std::string_view s, std::string_view separator)
static SingleDerivedPath parseWithSingle(
const Store & store, std::string_view s, std::string_view separator,
const ExperimentalFeatureSettings & xpSettings)
{
size_t n = s.find(separator);
size_t n = s.rfind(separator);
return n == s.npos
? (SingleDerivedPath) SingleDerivedPath::Opaque::parse(store, s)
: (SingleDerivedPath) SingleDerivedPath::Built::parse(store,
make_ref<SingleDerivedPath>(parseWithSingle(
store,
s.substr(0, n),
separator,
xpSettings)),
s.substr(n + 1),
xpSettings);
}
SingleDerivedPath SingleDerivedPath::parse(
const Store & store,
std::string_view s,
const ExperimentalFeatureSettings & xpSettings)
{
return parseWithSingle(store, s, "^", xpSettings);
}
SingleDerivedPath SingleDerivedPath::parseLegacy(
const Store & store,
std::string_view s,
const ExperimentalFeatureSettings & xpSettings)
{
return parseWithSingle(store, s, "!", xpSettings);
}
static DerivedPath parseWith(
const Store & store, std::string_view s, std::string_view separator,
const ExperimentalFeatureSettings & xpSettings)
{
size_t n = s.rfind(separator);
return n == s.npos
? (DerivedPath) DerivedPath::Opaque::parse(store, s)
: (DerivedPath) DerivedPath::Built::parse(store, s.substr(0, n), s.substr(n + 1));
: (DerivedPath) DerivedPath::Built::parse(store,
make_ref<SingleDerivedPath>(parseWithSingle(
store,
s.substr(0, n),
separator,
xpSettings)),
s.substr(n + 1),
xpSettings);
}
DerivedPath DerivedPath::parse(const Store & store, std::string_view s)
DerivedPath DerivedPath::parse(
const Store & store,
std::string_view s,
const ExperimentalFeatureSettings & xpSettings)
{
return parseWith(store, s, "^");
return parseWith(store, s, "^", xpSettings);
}
DerivedPath DerivedPath::parseLegacy(const Store & store, std::string_view s)
DerivedPath DerivedPath::parseLegacy(
const Store & store,
std::string_view s,
const ExperimentalFeatureSettings & xpSettings)
{
return parseWith(store, s, "!");
return parseWith(store, s, "!", xpSettings);
}
RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const
DerivedPath DerivedPath::fromSingle(const SingleDerivedPath & req)
{
RealisedPath::Set res;
std::visit(
overloaded{
[&](const BuiltPath::Opaque & p) { res.insert(p.path); },
[&](const BuiltPath::Built & p) {
auto drvHashes =
staticOutputHashes(store, store.readDerivation(p.drvPath));
for (auto& [outputName, outputPath] : p.outputs) {
if (experimentalFeatureSettings.isEnabled(
Xp::CaDerivations)) {
auto drvOutput = get(drvHashes, outputName);
if (!drvOutput)
throw Error(
"the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)",
store.printStorePath(p.drvPath), outputName);
auto thisRealisation = store.queryRealisation(
DrvOutput{*drvOutput, outputName});
assert(thisRealisation); // We’ve built it, so we must
// have the realisation
res.insert(*thisRealisation);
} else {
res.insert(outputPath);
}
}
return std::visit(overloaded {
[&](const SingleDerivedPath::Opaque & o) -> DerivedPath {
return o;
},
[&](const SingleDerivedPath::Built & b) -> DerivedPath {
return DerivedPath::Built {
.drvPath = b.drvPath,
.outputs = OutputsSpec::Names { b.output },
};
},
raw());
return res;
}, req.raw());
}
const StorePath & SingleDerivedPath::Built::getBaseStorePath() const
{
return drvPath->getBaseStorePath();
}
const StorePath & DerivedPath::Built::getBaseStorePath() const
{
return drvPath->getBaseStorePath();
}
template<typename DP>
static inline const StorePath & getBaseStorePath_(const DP & derivedPath)
{
return std::visit(overloaded {
[&](const typename DP::Built & bfd) -> auto & {
return bfd.drvPath->getBaseStorePath();
},
[&](const typename DP::Opaque & bo) -> auto & {
return bo.path;
},
}, derivedPath.raw());
}
const StorePath & SingleDerivedPath::getBaseStorePath() const
{
return getBaseStorePath_(*this);
}
const StorePath & DerivedPath::getBaseStorePath() const
{
return getBaseStorePath_(*this);
}
}

254
src/libstore/derived-path.hh
View file

@ -3,7 +3,6 @@
#include "util.hh"
#include "path.hh"
#include "realisation.hh"
#include "outputs-spec.hh"
#include "comparator.hh"
@ -25,28 +24,37 @@ class Store;
struct DerivedPathOpaque {
StorePath path;
nlohmann::json toJSON(ref<Store> store) const;
std::string to_string(const Store & store) const;
static DerivedPathOpaque parse(const Store & store, std::string_view);
nlohmann::json toJSON(const Store & store) const;
GENERATE_CMP(DerivedPathOpaque, me->path);
};
struct SingleDerivedPath;
/**
* A derived path that is built from a derivation
* A single derived path that is built from a derivation
*
* Built derived paths are pair of a derivation and some output names.
* They are evaluated by building the derivation, and then replacing the
* output names with the resulting outputs.
*
* Note that does mean a derived store paths evaluates to multiple
* opaque paths, which is sort of icky as expressions are supposed to
* evaluate to single values. Perhaps this should have just a single
* output name.
* Built derived paths are pair of a derivation and an output name. They are
* evaluated by building the derivation, and then taking the resulting output
* path of the given output name.
*/
struct DerivedPathBuilt {
StorePath drvPath;
OutputsSpec outputs;
struct SingleDerivedPathBuilt {
ref<SingleDerivedPath> drvPath;
std::string output;
/**
* Get the store path this is ultimately derived from (by realising
* and projecting outputs).
*
* Note that this is *not* a property of the store object being
* referred to, but just of this path --- how we happened to be
* referring to that store object. In other words, this means this
* function breaks "referential transparency". It should therefore
* be used only with great care.
*/
const StorePath & getBaseStorePath() const;
/**
* Uses `^` as the separator
@ -58,11 +66,139 @@ struct DerivedPathBuilt {
std::string to_string_legacy(const Store & store) const;
/**
* The caller splits on the separator, so it works for both variants.
*
* @param xpSettings Stop-gap to avoid globals during unit tests.
*/
static DerivedPathBuilt parse(const Store & store, std::string_view drvPath, std::string_view outputs);
nlohmann::json toJSON(ref<Store> store) const;
static SingleDerivedPathBuilt parse(
const Store & store, ref<SingleDerivedPath> drvPath,
std::string_view outputs,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
nlohmann::json toJSON(Store & store) const;
GENERATE_CMP(DerivedPathBuilt, me->drvPath, me->outputs);
DECLARE_CMP(SingleDerivedPathBuilt);
};
using _SingleDerivedPathRaw = std::variant<
DerivedPathOpaque,
SingleDerivedPathBuilt
>;
/**
* A "derived path" is a very simple sort of expression (not a Nix
* language expression! But an expression in a the general sense) that
* evaluates to (concrete) store path. It is either:
*
* - opaque, in which case it is just a concrete store path with
* possibly no known derivation
*
* - built, in which case it is a pair of a derivation path and an
* output name.
*/
struct SingleDerivedPath : _SingleDerivedPathRaw {
using Raw = _SingleDerivedPathRaw;
using Raw::Raw;
using Opaque = DerivedPathOpaque;
using Built = SingleDerivedPathBuilt;
inline const Raw & raw() const {
return static_cast<const Raw &>(*this);
}
/**
* Get the store path this is ultimately derived from (by realising
* and projecting outputs).
*
* Note that this is *not* a property of the store object being
* referred to, but just of this path --- how we happened to be
* referring to that store object. In other words, this means this
* function breaks "referential transparency". It should therefore
* be used only with great care.
*/
const StorePath & getBaseStorePath() const;
/**
* Uses `^` as the separator
*/
std::string to_string(const Store & store) const;
/**
* Uses `!` as the separator
*/
std::string to_string_legacy(const Store & store) const;
/**
* Uses `^` as the separator
*
* @param xpSettings Stop-gap to avoid globals during unit tests.
*/
static SingleDerivedPath parse(
const Store & store,
std::string_view,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
/**
* Uses `!` as the separator
*
* @param xpSettings Stop-gap to avoid globals during unit tests.
*/
static SingleDerivedPath parseLegacy(
const Store & store,
std::string_view,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
nlohmann::json toJSON(Store & store) const;
};
static inline ref<SingleDerivedPath> makeConstantStorePathRef(StorePath drvPath)
{
return make_ref<SingleDerivedPath>(SingleDerivedPath::Opaque { drvPath });
}
/**
* A set of derived paths that are built from a derivation
*
* Built derived paths are pair of a derivation and some output names.
* They are evaluated by building the derivation, and then replacing the
* output names with the resulting outputs.
*
* Note that does mean a derived store paths evaluates to multiple
* opaque paths, which is sort of icky as expressions are supposed to
* evaluate to single values. Perhaps this should have just a single
* output name.
*/
struct DerivedPathBuilt {
ref<SingleDerivedPath> drvPath;
OutputsSpec outputs;
/**
* Get the store path this is ultimately derived from (by realising
* and projecting outputs).
*
* Note that this is *not* a property of the store object being
* referred to, but just of this path --- how we happened to be
* referring to that store object. In other words, this means this
* function breaks "referential transparency". It should therefore
* be used only with great care.
*/
const StorePath & getBaseStorePath() const;
/**
* Uses `^` as the separator
*/
std::string to_string(const Store & store) const;
/**
* Uses `!` as the separator
*/
std::string to_string_legacy(const Store & store) const;
/**
* The caller splits on the separator, so it works for both variants.
*
* @param xpSettings Stop-gap to avoid globals during unit tests.
*/
static DerivedPathBuilt parse(
const Store & store, ref<SingleDerivedPath>,
std::string_view,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
nlohmann::json toJSON(Store & store) const;
DECLARE_CMP(DerivedPathBuilt);
};
using _DerivedPathRaw = std::variant<
@ -72,13 +208,13 @@ using _DerivedPathRaw = std::variant<
/**
* A "derived path" is a very simple sort of expression that evaluates
* to (concrete) store path. It is either:
* to one or more (concrete) store paths. It is either:
*
* - opaque, in which case it is just a concrete store path with
* - opaque, in which case it is just a single concrete store path with
* possibly no known derivation
*
* - built, in which case it is a pair of a derivation path and an
* output name.
* - built, in which case it is a pair of a derivation path and some
* output names.
*/
struct DerivedPath : _DerivedPathRaw {
using Raw = _DerivedPathRaw;
@ -91,6 +227,18 @@ struct DerivedPath : _DerivedPathRaw {
return static_cast<const Raw &>(*this);
}
/**
* Get the store path this is ultimately derived from (by realising
* and projecting outputs).
*
* Note that this is *not* a property of the store object being
* referred to, but just of this path --- how we happened to be
* referring to that store object. In other words, this means this
* function breaks "referential transparency". It should therefore
* be used only with great care.
*/
const StorePath & getBaseStorePath() const;
/**
* Uses `^` as the separator
*/
@ -101,55 +249,43 @@ struct DerivedPath : _DerivedPathRaw {
std::string to_string_legacy(const Store & store) const;
/**
* Uses `^` as the separator
*
* @param xpSettings Stop-gap to avoid globals during unit tests.
*/
static DerivedPath parse(const Store & store, std::string_view);
static DerivedPath parse(
const Store & store,
std::string_view,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
/**
* Uses `!` as the separator
*/
static DerivedPath parseLegacy(const Store & store, std::string_view);
};
/**
* A built derived path with hints in the form of optional concrete output paths.
*
* See 'BuiltPath' for more an explanation.
* @param xpSettings Stop-gap to avoid globals during unit tests.
*/
struct BuiltPathBuilt {
StorePath drvPath;
std::map<std::string, StorePath> outputs;
nlohmann::json toJSON(ref<Store> store) const;
static BuiltPathBuilt parse(const Store & store, std::string_view);
GENERATE_CMP(BuiltPathBuilt, me->drvPath, me->outputs);
};
using _BuiltPathRaw = std::variant<
DerivedPath::Opaque,
BuiltPathBuilt
>;
static DerivedPath parseLegacy(
const Store & store,
std::string_view,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
/**
* A built path. Similar to a DerivedPath, but enriched with the corresponding
* output path(s).
* Convert a `SingleDerivedPath` to a `DerivedPath`.
*/
struct BuiltPath : _BuiltPathRaw {
using Raw = _BuiltPathRaw;
using Raw::Raw;
using Opaque = DerivedPathOpaque;
using Built = BuiltPathBuilt;
inline const Raw & raw() const {
return static_cast<const Raw &>(*this);
}
StorePathSet outPaths() const;
RealisedPath::Set toRealisedPaths(Store & store) const;
static DerivedPath fromSingle(const SingleDerivedPath &);
nlohmann::json toJSON(Store & store) const;
};
typedef std::vector<DerivedPath> DerivedPaths;
typedef std::vector<BuiltPath> BuiltPaths;
/**
* Used by various parser functions to require experimental features as
* needed.
*
* Somewhat unfortunate this cannot just be an implementation detail for
* this module.
*
* @param xpSettings Stop-gap to avoid globals during unit tests.
*/
void drvRequireExperiment(
const SingleDerivedPath & drv,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
}

19
src/libstore/downstream-placeholder.cc
View file

@ -11,8 +11,10 @@ std::string DownstreamPlaceholder::render() const
DownstreamPlaceholder DownstreamPlaceholder::unknownCaOutput(
const StorePath & drvPath,
std::string_view outputName)
std::string_view outputName,
const ExperimentalFeatureSettings & xpSettings)
{
xpSettings.require(Xp::CaDerivations);
auto drvNameWithExtension = drvPath.name();
auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4);
auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName);
@ -36,4 +38,19 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation(
};
}
DownstreamPlaceholder DownstreamPlaceholder::fromSingleDerivedPathBuilt(
const SingleDerivedPath::Built & b)
{
return std::visit(overloaded {
[&](const SingleDerivedPath::Opaque & o) {
return DownstreamPlaceholder::unknownCaOutput(o.path, b.output);
},
[&](const SingleDerivedPath::Built & b2) {
return DownstreamPlaceholder::unknownDerivation(
DownstreamPlaceholder::fromSingleDerivedPathBuilt(b2),
b.output);
},
}, b.drvPath->raw());
}
}

17
src/libstore/downstream-placeholder.hh
View file

@ -3,6 +3,7 @@
#include "hash.hh"
#include "path.hh"
#include "derived-path.hh"
namespace nix {
@ -52,10 +53,13 @@ public:
*
* The derivation itself is known (we have a store path for it), but
* the output doesn't yet have a known store path.
*
* @param xpSettings Stop-gap to avoid globals during unit tests.
*/
static DownstreamPlaceholder unknownCaOutput(
const StorePath & drvPath,
std::string_view outputName);
std::string_view outputName,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
/**
* Create a placehold for the output of an unknown derivation.
@ -70,6 +74,17 @@ public:
const DownstreamPlaceholder & drvPlaceholder,
std::string_view outputName,
const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);
/**
* Convenience constructor that handles both cases (unknown
* content-addressed output and unknown derivation), delegating as
* needed to `unknownCaOutput` and `unknownDerivation`.
*
* Recursively builds up a placeholder from a
* `SingleDerivedPath::Built.drvPath` chain.
*/
static DownstreamPlaceholder fromSingleDerivedPathBuilt(
const SingleDerivedPath::Built & built);
};
}

35
src/libstore/gc-store.hh
View file

@ -71,19 +71,36 @@ struct GCResults
};
/**
* Mix-in class for \ref Store "stores" which expose a notion of garbage
* collection.
*
* Garbage collection will allow deleting paths which are not
* transitively "rooted".
*
* The notion of GC roots actually not part of this class.
*
* - The base `Store` class has `Store::addTempRoot()` because for a store
* that doesn't support garbage collection at all, a temporary GC root is
* safely implementable as no-op.
*
* @todo actually this is not so good because stores are *views*.
* Some views have only a no-op temp roots even though others to the
* same store allow triggering GC. For instance one can't add a root
* over ssh, but that doesn't prevent someone from gc-ing that store
* accesed via SSH locally).
*
* - The derived `LocalFSStore` class has `LocalFSStore::addPermRoot`,
* which is not part of this class because it relies on the notion of
* an ambient file system. There are stores (`ssh-ng://`, for one),
* that *do* support garbage collection but *don't* expose any file
* system, and `LocalFSStore::addPermRoot` thus does not make sense
* for them.
*/
struct GcStore : public virtual Store
{
inline static std::string operationName = "Garbage collection";
/**
* Add an indirect root, which is merely a symlink to `path` from
* `/nix/var/nix/gcroots/auto/<hash of path>`. `path` is supposed
* to be a symlink to a store path. The garbage collector will
* automatically remove the indirect root when it finds that
* `path` has disappeared.
*/
virtual void addIndirectRoot(const Path & path) = 0;
/**
* Find the roots of the garbage collector. Each root is a pair
* `(link, storepath)` where `link` is the path of the symlink

3
src/libstore/gc.cc
View file

@ -1,7 +1,6 @@
#include "derivations.hh"
#include "globals.hh"
#include "local-store.hh"
#include "local-fs-store.hh"
#include "finally.hh"
#include <functional>
@ -50,7 +49,7 @@ void LocalStore::addIndirectRoot(const Path & path)
}
Path LocalFSStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot)
Path IndirectRootStore::addPermRoot(const StorePath & storePath, const Path & _gcRoot)
{
Path gcRoot(canonPath(_gcRoot));

74
src/libstore/globals.hh
View file

@ -193,18 +193,24 @@ public:
Setting<std::string> thisSystem{
this, SYSTEM, "system",
R"(
This option specifies the canonical Nix system name of the current
installation, such as `i686-linux` or `x86_64-darwin`. Nix can only
build derivations whose `system` attribute equals the value
specified here. In general, it never makes sense to modify this
value from its default, since you can use it to ‘lie’ about the
platform you are building on (e.g., perform a Mac OS build on a
Linux machine; the result would obviously be wrong). It only makes
sense if the Nix binaries can run on multiple platforms, e.g.,
‘universal binaries’ that run on `x86_64-linux` and `i686-linux`.
The system type of the current Nix installation.
Nix will only build a given [derivation](@docroot@/language/derivations.md) locally when its `system` attribute equals any of the values specified here or in [`extra-platforms`](#conf-extra-platforms).
It defaults to the canonical Nix system name detected by `configure`
at build time.
The default value is set when Nix itself is compiled for the system it will run on.
The following system types are widely used, as [Nix is actively supported on these platforms](@docroot@/contributing/hacking.md#platforms):
- `x86_64-linux`
- `x86_64-darwin`
- `i686-linux`
- `aarch64-linux`
- `aarch64-darwin`
- `armv6l-linux`
- `armv7l-linux`
In general, you do not have to modify this setting.
While you can force Nix to run a Darwin-specific `builder` executable on a Linux machine, the result would obviously be wrong.
This value is available in the Nix language as [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem).
)"};
Setting<time_t> maxSilentTime{
@ -524,6 +530,24 @@ public:
Setting<bool> sandboxFallback{this, true, "sandbox-fallback",
"Whether to disable sandboxing when the kernel doesn't allow it."};
Setting<bool> requireDropSupplementaryGroups{this, getuid() == 0, "require-drop-supplementary-groups",
R"(
Following the principle of least privilege,
Nix will attempt to drop supplementary groups when building with sandboxing.
However this can fail under some circumstances.
For example, if the user lacks the `CAP_SETGID` capability.
Search `setgroups(2)` for `EPERM` to find more detailed information on this.
If you encounter such a failure, setting this option to `false` will let you ignore it and continue.
But before doing so, you should consider the security implications carefully.
Not dropping supplementary groups means the build sandbox will be less restricted than intended.
This option defaults to `true` when the user is root
(since `root` usually has permissions to call setgroups)
and `false` otherwise.
)"};
#if __linux__
Setting<std::string> sandboxShmSize{
this, "50%", "sandbox-dev-shm-size",
@ -652,18 +676,20 @@ public:
getDefaultExtraPlatforms(),
"extra-platforms",
R"(
Platforms other than the native one which this machine is capable of
building for. This can be useful for supporting additional
architectures on compatible machines: i686-linux can be built on
x86\_64-linux machines (and the default for this setting reflects
this); armv7 is backwards-compatible with armv6 and armv5tel; some
aarch64 machines can also natively run 32-bit ARM code; and
qemu-user may be used to support non-native platforms (though this
may be slow and buggy). Most values for this are not enabled by
default because build systems will often misdetect the target
platform and generate incompatible code, so you may wish to
cross-check the results of using this option against proper
natively-built versions of your derivations.
System types of executables that can be run on this machine.
Nix will only build a given [derivation](@docroot@/language/derivations.md) locally when its `system` attribute equals any of the values specified here or in the [`system` option](#conf-system).
Setting this can be useful to build derivations locally on compatible machines:
- `i686-linux` executables can be run on `x86_64-linux` machines (set by default)
- `x86_64-darwin` executables can be run on macOS `aarch64-darwin` with Rosetta 2 (set by default where applicable)
- `armv6` and `armv5tel` executables can be run on `armv7`
- some `aarch64` machines can also natively run 32-bit ARM code
- `qemu-user` may be used to support non-native platforms (though this
may be slow and buggy)
Build systems will usually detect the target platform to be the current physical system and therefore produce machine code incompatible with what may be intended in the derivation.
You should design your derivation's `builder` accordingly and cross-check the results when using this option against natively-built versions of your derivation.
)", {}, false};
Setting<StringSet> systemFeatures{
@ -992,7 +1018,7 @@ public:
| `~/.nix-defexpr` | `$XDG_STATE_HOME/nix/defexpr` |
| `~/.nix-channels` | `$XDG_STATE_HOME/nix/channels` |
If you already have Nix installed and are using [profiles](@docroot@/package-management/profiles.md) or [channels](@docroot@/package-management/channels.md), you should migrate manually when you enable this option.
If you already have Nix installed and are using [profiles](@docroot@/package-management/profiles.md) or [channels](@docroot@/command-ref/nix-channel.md), you should migrate manually when you enable this option.
If `$XDG_STATE_HOME` is not set, use `$HOME/.local/state/nix` instead of `$XDG_STATE_HOME/nix`.
This can be achieved with the following shell commands:

48
src/libstore/indirect-root-store.hh Normal file
View file

@ -0,0 +1,48 @@
#pragma once
///@file
#include "local-fs-store.hh"
namespace nix {
/**
* Mix-in class for implementing permanent roots as a pair of a direct
* (strong) reference and indirect weak reference to the first
* reference.
*
* See methods for details on the operations it represents.
*/
struct IndirectRootStore : public virtual LocalFSStore
{
inline static std::string operationName = "Indirect GC roots registration";
/**
* Implementation of `LocalFSStore::addPermRoot` where the permanent
* root is a pair of
*
* - The user-facing symlink which all implementations must create
*
* - An additional weak reference known as the "indirect root" that
* points to that symlink.
*
* The garbage collector will automatically remove the indirect root
* when it finds that the symlink has disappeared.
*
* The implementation of this method is concrete, but it delegates
* to `addIndirectRoot()` which is abstract.
*/
Path addPermRoot(const StorePath & storePath, const Path & gcRoot) override final;
/**
* Add an indirect root, which is a weak reference to the
* user-facing symlink created by `addPermRoot()`.
*
* @param path user-facing and user-controlled symlink to a store
* path.
*
* The form this weak-reference takes is implementation-specific.
*/
virtual void addIndirectRoot(const Path & path) = 0;
};
}

3
src/libstore/legacy-ssh-store.cc
View file

@ -358,6 +358,9 @@ public:
[&](const StorePath & drvPath) {
throw Error("wanted to fetch '%s' but the legacy ssh protocol doesn't support merely substituting drv files via the build paths command. It would build them instead. Try using ssh-ng://", printStorePath(drvPath));
},
[&](std::monostate) {
throw Error("wanted build derivation that is itself a build product, but the legacy ssh protocol doesn't support that. Try using ssh-ng://");
},
}, sOrDrvPath);
}
conn->to << ss;

16
src/libstore/local-fs-store.hh
View file

@ -40,6 +40,7 @@ class LocalFSStore : public virtual LocalFSStoreConfig,
public virtual LogStore
{
public:
inline static std::string operationName = "Local Filesystem Store";
const static std::string drvsLogDir;
@ -49,9 +50,20 @@ public:
ref<FSAccessor> getFSAccessor() override;
/**
* Register a permanent GC root.
* Creates symlink from the `gcRoot` to the `storePath` and
* registers the `gcRoot` as a permanent GC root. The `gcRoot`
* symlink lives outside the store and is created and owned by the
* user.
*
* @param gcRoot The location of the symlink.
*
* @param storePath The store object being rooted. The symlink will
* point to `toRealPath(store.printStorePath(storePath))`.
*
* How the permanent GC root corresponding to this symlink is
* managed is implementation-specific.
*/
Path addPermRoot(const StorePath & storePath, const Path & gcRoot);
virtual Path addPermRoot(const StorePath & storePath, const Path & gcRoot) = 0;
virtual Path getRealStoreDir() { return realStoreDir; }

132
src/libstore/local-store.cc
View file

@ -1022,10 +1022,9 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
std::map<std::string, std::optional<StorePath>>
LocalStore::queryPartialDerivationOutputMap(const StorePath & path_)
LocalStore::queryStaticPartialDerivationOutputMap(const StorePath & path)
{
auto path = path_;
auto outputs = retrySQLite<std::map<std::string, std::optional<StorePath>>>([&]() {
return retrySQLite<std::map<std::string, std::optional<StorePath>>>([&]() {
auto state(_state.lock());
std::map<std::string, std::optional<StorePath>> outputs;
uint64_t drvId;
@ -1037,21 +1036,6 @@ LocalStore::queryPartialDerivationOutputMap(const StorePath & path_)
return outputs;
});
if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations))
return outputs;
auto drv = readInvalidDerivation(path);
auto drvHashes = staticOutputHashes(*this, drv);
for (auto& [outputName, hash] : drvHashes) {
auto realisation = queryRealisation(DrvOutput{hash, outputName});
if (realisation)
outputs.insert_or_assign(outputName, realisation->outPath);
else
outputs.insert({outputName, std::nullopt});
}
return outputs;
}
std::optional<StorePath> LocalStore::queryPathFromHashPart(const std::string & hashPart)
@ -1212,6 +1196,15 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
if (checkSigs && pathInfoIsUntrusted(info))
throw Error("cannot add path '%s' because it lacks a signature by a trusted key", printStorePath(info.path));
/* In case we are not interested in reading the NAR: discard it. */
bool narRead = false;
Finally cleanup = [&]() {
if (!narRead) {
ParseSink sink;
parseDump(sink, source);
}
};
addTempRoot(info.path);
if (repair || !isValidPath(info.path)) {
@ -1236,6 +1229,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
TeeSource wrapperSource { source, hashSink };
narRead = true;
restorePath(realPath, wrapperSource);
auto hashResult = hashSink.finish();
@ -1249,27 +1243,17 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
printStorePath(info.path), info.narSize, hashResult.second);
if (info.ca) {
if (auto foHash = std::get_if<FixedOutputHash>(&info.ca->raw)) {
auto actualFoHash = hashCAPath(
foHash->method,
foHash->hash.type,
auto & specified = *info.ca;
auto actualHash = hashCAPath(
specified.method,
specified.hash.type,
info.path
);
if (foHash->hash != actualFoHash.hash) {
if (specified.hash != actualHash.hash) {
throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s",
printStorePath(info.path),
foHash->hash.to_string(Base32, true),
actualFoHash.hash.to_string(Base32, true));
}
}
if (auto textHash = std::get_if<TextHash>(&info.ca->raw)) {
auto actualTextHash = hashString(htSHA256, readFile(realPath));
if (textHash->hash != actualTextHash) {
throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s",
printStorePath(info.path),
textHash->hash.to_string(Base32, true),
actualTextHash.to_string(Base32, true));
}
specified.hash.to_string(Base32, true),
actualHash.hash.to_string(Base32, true));
}
}
@ -1349,10 +1333,8 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name
auto [hash, size] = hashSink->finish();
ContentAddressWithReferences desc = FixedOutputInfo {
.hash = {
.method = method,
.hash = hash,
},
.references = {
.others = references,
// caller is not capable of creating a self-reference, because this is content-addressed without modulus
@ -1428,8 +1410,8 @@ StorePath LocalStore::addTextToStore(
{
auto hash = hashString(htSHA256, s);
auto dstPath = makeTextPath(name, TextInfo {
{ .hash = hash },
references,
.hash = hash,
.references = references,
});
addTempRoot(dstPath);
@ -1459,7 +1441,10 @@ StorePath LocalStore::addTextToStore(
ValidPathInfo info { dstPath, narHash };
info.narSize = sink.s.size();
info.references = references;
info.ca = TextHash { .hash = hash };
info.ca = {
.method = TextIngestionMethod {},
.hash = hash,
};
registerValidPath(info);
}
@ -1524,17 +1509,33 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
auto fdGCLock = openGCLock();
FdLock gcLock(fdGCLock.get(), ltRead, true, "waiting for the big garbage collector lock...");
StringSet store;
for (auto & i : readDirectory(realStoreDir)) store.insert(i.name);
StorePathSet validPaths;
{
StorePathSet storePathsInStoreDir;
/* Why aren't we using `queryAllValidPaths`? Because that would
tell us about all the paths than the database knows about. Here we
want to know about all the store paths in the store directory,
regardless of what the database thinks.
We will end up cross-referencing these two sources of truth (the
database and the filesystem) in the loop below, in order to catch
invalid states.
*/
for (auto & i : readDirectory(realStoreDir)) {
try {
storePathsInStoreDir.insert({i.name});
} catch (BadStorePath &) { }
}
/* Check whether all valid paths actually exist. */
printInfo("checking path existence...");
StorePathSet validPaths;
PathSet done;
StorePathSet done;
for (auto & i : queryAllValidPaths())
verifyPath(printStorePath(i), store, done, validPaths, repair, errors);
verifyPath(i, storePathsInStoreDir, done, validPaths, repair, errors);
}
/* Optionally, check the content hashes (slow). */
if (checkContents) {
@ -1620,32 +1621,27 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
}
void LocalStore::verifyPath(const Path & pathS, const StringSet & store,
PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors)
void LocalStore::verifyPath(const StorePath & path, const StorePathSet & storePathsInStoreDir,
StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors)
{
checkInterrupt();
if (!done.insert(pathS).second) return;
if (!done.insert(path).second) return;
if (!isStorePath(pathS)) {
printError("path '%s' is not in the Nix store", pathS);
return;
}
auto path = parseStorePath(pathS);
if (!store.count(std::string(path.to_string()))) {
if (!storePathsInStoreDir.count(path)) {
/* Check any referrers first. If we can invalidate them
first, then we can invalidate this path as well. */
bool canInvalidate = true;
StorePathSet referrers; queryReferrers(path, referrers);
for (auto & i : referrers)
if (i != path) {
verifyPath(printStorePath(i), store, done, validPaths, repair, errors);
verifyPath(i, storePathsInStoreDir, done, validPaths, repair, errors);
if (validPaths.count(i))
canInvalidate = false;
}
auto pathS = printStorePath(path);
if (canInvalidate) {
printInfo("path '%s' disappeared, removing from database...", pathS);
auto state(_state.lock());
@ -1856,22 +1852,27 @@ void LocalStore::queryRealisationUncached(const DrvOutput & id,
}
}
FixedOutputHash LocalStore::hashCAPath(
const FileIngestionMethod & method, const HashType & hashType,
ContentAddress LocalStore::hashCAPath(
const ContentAddressMethod & method, const HashType & hashType,
const StorePath & path)
{
return hashCAPath(method, hashType, Store::toRealPath(path), path.hashPart());
}
FixedOutputHash LocalStore::hashCAPath(
const FileIngestionMethod & method,
ContentAddress LocalStore::hashCAPath(
const ContentAddressMethod & method,
const HashType & hashType,
const Path & path,
const std::string_view pathHash
)
{
HashModuloSink caSink ( hashType, std::string(pathHash) );
switch (method) {
std::visit(overloaded {
[&](const TextIngestionMethod &) {
readFile(path, caSink);
},
[&](const FileIngestionMethod & m2) {
switch (m2) {
case FileIngestionMethod::Recursive:
dumpPath(path, caSink);
break;
@ -1879,10 +1880,11 @@ FixedOutputHash LocalStore::hashCAPath(
readFile(path, caSink);
break;
}
auto hash = caSink.finish().first;
return FixedOutputHash{
},
}, method.raw);
return ContentAddress {
.method = method,
.hash = hash,
.hash = caSink.finish().first,
};
}

27
src/libstore/local-store.hh
View file

@ -5,8 +5,7 @@
#include "pathlocks.hh"
#include "store-api.hh"
#include "local-fs-store.hh"
#include "gc-store.hh"
#include "indirect-root-store.hh"
#include "sync.hh"
#include "util.hh"
@ -68,7 +67,9 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig
std::string doc() override;
};
class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore, public virtual GcStore
class LocalStore : public virtual LocalStoreConfig
, public virtual IndirectRootStore
, public virtual GcStore
{
private:
@ -165,7 +166,7 @@ public:
StorePathSet queryValidDerivers(const StorePath & path) override;
std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override;
std::map<std::string, std::optional<StorePath>> queryStaticPartialDerivationOutputMap(const StorePath & path) override;
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
@ -209,6 +210,12 @@ private:
public:
/**
* Implementation of IndirectRootStore::addIndirectRoot().
*
* The weak reference merely is a symlink to `path' from
* /nix/var/nix/gcroots/auto/<hash of `path'>.
*/
void addIndirectRoot(const Path & path) override;
private:
@ -307,8 +314,8 @@ private:
*/
void invalidatePathChecked(const StorePath & path);
void verifyPath(const Path & path, const StringSet & store,
PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors);
void verifyPath(const StorePath & path, const StorePathSet & store,
StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors);
std::shared_ptr<const ValidPathInfo> queryPathInfoInternal(State & state, const StorePath & path);
@ -345,13 +352,13 @@ private:
void signRealisation(Realisation &);
// XXX: Make a generic `Store` method
FixedOutputHash hashCAPath(
const FileIngestionMethod & method,
ContentAddress hashCAPath(
const ContentAddressMethod & method,
const HashType & hashType,
const StorePath & path);
FixedOutputHash hashCAPath(
const FileIngestionMethod & method,
ContentAddress hashCAPath(
const ContentAddressMethod & method,
const HashType & hashType,
const Path & path,
const std::string_view pathHash

2
src/libstore/make-content-addressed.cc
View file

@ -52,10 +52,8 @@ std::map<StorePath, StorePath> makeContentAddressed(
dstStore,
path.name(),
FixedOutputInfo {
.hash = {
.method = FileIngestionMethod::Recursive,
.hash = narModuloHash,
},
.references = std::move(refs),
},
Hash::dummy,

160
src/libstore/misc.cc
View file

@ -132,7 +132,7 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
}
for (auto & i : drv.inputDrvs)
pool.enqueue(std::bind(doPath, DerivedPath::Built { i.first, i.second }));
pool.enqueue(std::bind(doPath, DerivedPath::Built { makeConstantStorePathRef(i.first), i.second }));
};
auto checkOutput = [&](
@ -176,10 +176,18 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
std::visit(overloaded {
[&](const DerivedPath::Built & bfd) {
if (!isValidPath(bfd.drvPath)) {
auto drvPathP = std::get_if<DerivedPath::Opaque>(&*bfd.drvPath);
if (!drvPathP) {
// TODO make work in this case.
warn("Ignoring dynamic derivation %s while querying missing paths; not yet implemented", bfd.drvPath->to_string(*this));
return;
}
auto & drvPath = drvPathP->path;
if (!isValidPath(drvPath)) {
// FIXME: we could try to substitute the derivation.
auto state(state_.lock());
state->unknown.insert(bfd.drvPath);
state->unknown.insert(drvPath);
return;
}
@ -187,7 +195,7 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
/* true for regular derivations, and CA derivations for which we
have a trust mapping for all wanted outputs. */
auto knownOutputPaths = true;
for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(bfd.drvPath)) {
for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(drvPath)) {
if (!pathOpt) {
knownOutputPaths = false;
break;
@ -197,15 +205,45 @@ void Store::queryMissing(const std::vector<DerivedPath> & targets,
}
if (knownOutputPaths && invalid.empty()) return;
auto drv = make_ref<Derivation>(derivationFromPath(bfd.drvPath));
ParsedDerivation parsedDrv(StorePath(bfd.drvPath), *drv);
auto drv = make_ref<Derivation>(derivationFromPath(drvPath));
ParsedDerivation parsedDrv(StorePath(drvPath), *drv);
if (!knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
experimentalFeatureSettings.require(Xp::CaDerivations);
// If there are unknown output paths, attempt to find if the
// paths are known to substituters through a realisation.
auto outputHashes = staticOutputHashes(*this, *drv);
knownOutputPaths = true;
for (auto [outputName, hash] : outputHashes) {
if (!bfd.outputs.contains(outputName))
continue;
bool found = false;
for (auto &sub : getDefaultSubstituters()) {
auto realisation = sub->queryRealisation({hash, outputName});
if (!realisation)
continue;
found = true;
if (!isValidPath(realisation->outPath))
invalid.insert(realisation->outPath);
break;
}
if (!found) {
// Some paths did not have a realisation, this must be built.
knownOutputPaths = false;
break;
}
}
}
if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size()));
for (auto & output : invalid)
pool.enqueue(std::bind(checkOutput, bfd.drvPath, drv, output, drvState));
pool.enqueue(std::bind(checkOutput, drvPath, drv, output, drvState));
} else
mustBuildDrv(bfd.drvPath, *drv);
mustBuildDrv(drvPath, *drv);
},
[&](const DerivedPath::Opaque & bo) {
@ -310,45 +348,91 @@ std::map<DrvOutput, StorePath> drvOutputReferences(
OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd, Store * evalStore_)
{
auto & evalStore = evalStore_ ? *evalStore_ : store;
auto drvPath = resolveDerivedPath(store, *bfd.drvPath, evalStore_);
OutputPathMap outputs;
auto drv = evalStore.readDerivation(bfd.drvPath);
auto outputHashes = staticOutputHashes(store, drv);
auto drvOutputs = drv.outputsAndOptPaths(store);
auto outputNames = std::visit(overloaded {
auto outputsOpt_ = store.queryPartialDerivationOutputMap(drvPath, evalStore_);
auto outputsOpt = std::visit(overloaded {
[&](const OutputsSpec::All &) {
StringSet names;
for (auto & [outputName, _] : drv.outputs)
names.insert(outputName);
return names;
// Keep all outputs
return std::move(outputsOpt_);
},
[&](const OutputsSpec::Names & names) {
return static_cast<std::set<std::string>>(names);
},
}, bfd.outputs.raw());
for (auto & output : outputNames) {
auto outputHash = get(outputHashes, output);
if (!outputHash)
// Get just those mentioned by name
std::map<std::string, std::optional<StorePath>> outputsOpt;
for (auto & output : names) {
auto * pOutputPathOpt = get(outputsOpt_, output);
if (!pOutputPathOpt)
throw Error(
"the derivation '%s' doesn't have an output named '%s'",
store.printStorePath(bfd.drvPath), output);
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
DrvOutput outputId { *outputHash, output };
auto realisation = store.queryRealisation(outputId);
if (!realisation)
throw MissingRealisation(outputId);
outputs.insert_or_assign(output, realisation->outPath);
} else {
// If ca-derivations isn't enabled, assume that
// the output path is statically known.
auto drvOutput = get(drvOutputs, output);
assert(drvOutput);
assert(drvOutput->second);
outputs.insert_or_assign(output, *drvOutput->second);
bfd.drvPath->to_string(store), output);
outputsOpt.insert_or_assign(output, std::move(*pOutputPathOpt));
}
return outputsOpt;
},
}, bfd.outputs.raw());
OutputPathMap outputs;
for (auto & [outputName, outputPathOpt] : outputsOpt) {
if (!outputPathOpt)
throw MissingRealisation(bfd.drvPath->to_string(store), outputName);
auto & outputPath = *outputPathOpt;
outputs.insert_or_assign(outputName, outputPath);
}
return outputs;
}
StorePath resolveDerivedPath(Store & store, const SingleDerivedPath & req, Store * evalStore_)
{
auto & evalStore = evalStore_ ? *evalStore_ : store;
return std::visit(overloaded {
[&](const SingleDerivedPath::Opaque & bo) {
return bo.path;
},
[&](const SingleDerivedPath::Built & bfd) {
auto drvPath = resolveDerivedPath(store, *bfd.drvPath, evalStore_);
auto outputPaths = evalStore.queryPartialDerivationOutputMap(drvPath, evalStore_);
if (outputPaths.count(bfd.output) == 0)
throw Error("derivation '%s' does not have an output named '%s'",
store.printStorePath(drvPath), bfd.output);
auto & optPath = outputPaths.at(bfd.output);
if (!optPath)
throw Error("'%s' does not yet map to a known concrete store path",
bfd.to_string(store));
return *optPath;
},
}, req.raw());
}
OutputPathMap resolveDerivedPath(Store & store, const DerivedPath::Built & bfd)
{
auto drvPath = resolveDerivedPath(store, *bfd.drvPath);
auto outputMap = store.queryDerivationOutputMap(drvPath);
auto outputsLeft = std::visit(overloaded {
[&](const OutputsSpec::All &) {
return StringSet {};
},
[&](const OutputsSpec::Names & names) {
return static_cast<StringSet>(names);
},
}, bfd.outputs.raw());
for (auto iter = outputMap.begin(); iter != outputMap.end();) {
auto & outputName = iter->first;
if (bfd.outputs.contains(outputName)) {
outputsLeft.erase(outputName);
++iter;
} else {
iter = outputMap.erase(iter);
}
}
if (!outputsLeft.empty())
throw Error("derivation '%s' does not have an outputs %s",
store.printStorePath(drvPath),
concatStringsSep(", ", quoteStrings(std::get<OutputsSpec::Names>(bfd.outputs))));
return outputMap;
}
}

21
src/libstore/path-info.cc
View file

@ -29,14 +29,14 @@ std::optional<ContentAddressWithReferences> ValidPathInfo::contentAddressWithRef
return std::nullopt;
return std::visit(overloaded {
[&](const TextHash & th) -> ContentAddressWithReferences {
[&](const TextIngestionMethod &) -> ContentAddressWithReferences {
assert(references.count(path) == 0);
return TextInfo {
.hash = th,
.hash = ca->hash,
.references = references,
};
},
[&](const FixedOutputHash & foh) -> ContentAddressWithReferences {
[&](const FileIngestionMethod & m2) -> ContentAddressWithReferences {
auto refs = references;
bool hasSelfReference = false;
if (refs.count(path)) {
@ -44,14 +44,15 @@ std::optional<ContentAddressWithReferences> ValidPathInfo::contentAddressWithRef
refs.erase(path);
}
return FixedOutputInfo {
.hash = foh,
.method = m2,
.hash = ca->hash,
.references = {
.others = std::move(refs),
.self = hasSelfReference,
},
};
},
}, ca->raw);
}, ca->method.raw);
}
bool ValidPathInfo::isContentAddressed(const Store & store) const
@ -110,13 +111,19 @@ ValidPathInfo::ValidPathInfo(
std::visit(overloaded {
[this](TextInfo && ti) {
this->references = std::move(ti.references);
this->ca = std::move((TextHash &&) ti);
this->ca = ContentAddress {
.method = TextIngestionMethod {},
.hash = std::move(ti.hash),
};
},
[this](FixedOutputInfo && foi) {
this->references = std::move(foi.references.others);
if (foi.references.self)
this->references.insert(path);
this->ca = std::move((FixedOutputHash &&) foi);
this->ca = ContentAddress {
.method = std::move(foi.method),
.hash = std::move(foi.hash),
};
},
}, std::move(ca).raw);
}

25
src/libstore/path-with-outputs.cc
View file

@ -16,10 +16,16 @@ std::string StorePathWithOutputs::to_string(const Store & store) const
DerivedPath StorePathWithOutputs::toDerivedPath() const
{
if (!outputs.empty()) {
return DerivedPath::Built { path, OutputsSpec::Names { outputs } };
return DerivedPath::Built {
.drvPath = makeConstantStorePathRef(path),
.outputs = OutputsSpec::Names { outputs },
};
} else if (path.isDerivation()) {
assert(outputs.empty());
return DerivedPath::Built { path, OutputsSpec::All { } };
return DerivedPath::Built {
.drvPath = makeConstantStorePathRef(path),
.outputs = OutputsSpec::All { },
};
} else {
return DerivedPath::Opaque { path };
}
@ -34,19 +40,21 @@ std::vector<DerivedPath> toDerivedPaths(const std::vector<StorePathWithOutputs>
}
std::variant<StorePathWithOutputs, StorePath> StorePathWithOutputs::tryFromDerivedPath(const DerivedPath & p)
StorePathWithOutputs::ParseResult StorePathWithOutputs::tryFromDerivedPath(const DerivedPath & p)
{
return std::visit(overloaded {
[&](const DerivedPath::Opaque & bo) -> std::variant<StorePathWithOutputs, StorePath> {
[&](const DerivedPath::Opaque & bo) -> StorePathWithOutputs::ParseResult {
if (bo.path.isDerivation()) {
// drv path gets interpreted as "build", not "get drv file itself"
return bo.path;
}
return StorePathWithOutputs { bo.path };
},
[&](const DerivedPath::Built & bfd) -> std::variant<StorePathWithOutputs, StorePath> {
[&](const DerivedPath::Built & bfd) -> StorePathWithOutputs::ParseResult {
return std::visit(overloaded {
[&](const SingleDerivedPath::Opaque & bo) -> StorePathWithOutputs::ParseResult {
return StorePathWithOutputs {
.path = bfd.drvPath,
.path = bo.path,
// Use legacy encoding of wildcard as empty set
.outputs = std::visit(overloaded {
[&](const OutputsSpec::All &) -> StringSet {
@ -58,6 +66,11 @@ std::variant<StorePathWithOutputs, StorePath> StorePathWithOutputs::tryFromDeriv
}, bfd.outputs.raw()),
};
},
[&](const SingleDerivedPath::Built &) -> StorePathWithOutputs::ParseResult {
return std::monostate {};
},
}, bfd.drvPath->raw());
},
}, p.raw());
}

4
src/libstore/path-with-outputs.hh
View file

@ -23,7 +23,9 @@ struct StorePathWithOutputs
DerivedPath toDerivedPath() const;
static std::variant<StorePathWithOutputs, StorePath> tryFromDerivedPath(const DerivedPath &);
typedef std::variant<StorePathWithOutputs, StorePath, std::monostate> ParseResult;
static StorePathWithOutputs::ParseResult tryFromDerivedPath(const DerivedPath &);
};
std::vector<DerivedPath> toDerivedPaths(const std::vector<StorePathWithOutputs>);

9
src/libstore/realisation.hh
View file

@ -5,6 +5,7 @@
#include "hash.hh"
#include "path.hh"
#include "derived-path.hh"
#include <nlohmann/json_fwd.hpp>
#include "comparator.hh"
#include "crypto.hh"
@ -143,9 +144,13 @@ class MissingRealisation : public Error
{
public:
MissingRealisation(DrvOutput & outputId)
: Error( "cannot operate on an output of the "
: MissingRealisation(outputId.outputName, outputId.strHash())
{}
MissingRealisation(std::string_view drv, std::string outputName)
: Error( "cannot operate on output '%s' of the "
"unbuilt derivation '%s'",
outputId.to_string())
outputName,
drv)
{}
};

31
src/libstore/remote-store-connection.hh
View file

@ -1,5 +1,6 @@
#include "remote-store.hh"
#include "worker-protocol.hh"
#include "pool.hh"
namespace nix {
@ -94,4 +95,34 @@ struct RemoteStore::Connection
std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0, bool flush = true);
};
/**
* A wrapper around Pool<RemoteStore::Connection>::Handle that marks
* the connection as bad (causing it to be closed) if a non-daemon
* exception is thrown before the handle is closed. Such an exception
* causes a deviation from the expected protocol and therefore a
* desynchronization between the client and daemon.
*/
struct RemoteStore::ConnectionHandle
{
Pool<RemoteStore::Connection>::Handle handle;
bool daemonException = false;
ConnectionHandle(Pool<RemoteStore::Connection>::Handle && handle)
: handle(std::move(handle))
{ }
ConnectionHandle(ConnectionHandle && h)
: handle(std::move(h.handle))
{ }
~ConnectionHandle();
RemoteStore::Connection & operator * () { return *handle; }
RemoteStore::Connection * operator -> () { return &*handle; }
void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true);
void withFramedSink(std::function<void(Sink & sink)> fun);
};
}

87
src/libstore/remote-store.cc
View file

@ -159,25 +159,7 @@ void RemoteStore::setOptions(Connection & conn)
}
/* A wrapper around Pool<RemoteStore::Connection>::Handle that marks
the connection as bad (causing it to be closed) if a non-daemon
exception is thrown before the handle is closed. Such an exception
causes a deviation from the expected protocol and therefore a
desynchronization between the client and daemon. */
struct ConnectionHandle
{
Pool<RemoteStore::Connection>::Handle handle;
bool daemonException = false;
ConnectionHandle(Pool<RemoteStore::Connection>::Handle && handle)
: handle(std::move(handle))
{ }
ConnectionHandle(ConnectionHandle && h)
: handle(std::move(h.handle))
{ }
~ConnectionHandle()
RemoteStore::ConnectionHandle::~ConnectionHandle()
{
if (!daemonException && std::uncaught_exceptions()) {
handle.markBad();
@ -185,10 +167,7 @@ struct ConnectionHandle
}
}
RemoteStore::Connection * operator -> () { return &*handle; }
RemoteStore::Connection & operator * () { return *handle; }
void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true)
void RemoteStore::ConnectionHandle::processStderr(Sink * sink, Source * source, bool flush)
{
auto ex = handle->processStderr(sink, source, flush);
if (ex) {
@ -197,11 +176,8 @@ struct ConnectionHandle
}
}
void withFramedSink(std::function<void(Sink & sink)> fun);
};
ConnectionHandle RemoteStore::getConnection()
RemoteStore::ConnectionHandle RemoteStore::getConnection()
{
return ConnectionHandle(connections->get());
}
@ -378,27 +354,36 @@ StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
}
std::map<std::string, std::optional<StorePath>> RemoteStore::queryPartialDerivationOutputMap(const StorePath & path)
std::map<std::string, std::optional<StorePath>> RemoteStore::queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore_)
{
if (GET_PROTOCOL_MINOR(getProtocol()) >= 0x16) {
if (!evalStore_) {
auto conn(getConnection());
conn->to << WorkerProto::Op::QueryDerivationOutputMap << printStorePath(path);
conn.processStderr();
return WorkerProto::Serialise<std::map<std::string, std::optional<StorePath>>>::read(*this, *conn);
} else {
auto & evalStore = *evalStore_;
auto outputs = evalStore.queryStaticPartialDerivationOutputMap(path);
// union with the first branch overriding the statically-known ones
// when non-`std::nullopt`.
for (auto && [outputName, optPath] : queryPartialDerivationOutputMap(path, nullptr)) {
if (optPath)
outputs.insert_or_assign(std::move(outputName), std::move(optPath));
else
outputs.insert({std::move(outputName), std::nullopt});
}
return outputs;
}
} else {
auto & evalStore = evalStore_ ? *evalStore_ : *this;
// Fallback for old daemon versions.
// For floating-CA derivations (and their co-dependencies) this is an
// under-approximation as it only returns the paths that can be inferred
// from the derivation itself (and not the ones that are known because
// the have been built), but as old stores don't handle floating-CA
// derivations this shouldn't matter
auto derivation = readDerivation(path);
auto outputsWithOptPaths = derivation.outputsAndOptPaths(*this);
std::map<std::string, std::optional<StorePath>> ret;
for (auto & [outputName, outputAndPath] : outputsWithOptPaths) {
ret.emplace(outputName, outputAndPath.second);
}
return ret;
return evalStore.queryStaticPartialDerivationOutputMap(path);
}
}
@ -671,6 +656,9 @@ static void writeDerivedPaths(RemoteStore & store, RemoteStore::Connection & con
GET_PROTOCOL_MAJOR(conn.daemonVersion),
GET_PROTOCOL_MINOR(conn.daemonVersion));
},
[&](std::monostate) {
throw Error("wanted to build a derivation that is itself a build product, but the legacy 'ssh://' protocol doesn't support that. Try using 'ssh-ng://'");
},
}, sOrDrvPath);
}
conn.to << ss;
@ -685,9 +673,16 @@ void RemoteStore::copyDrvsFromEvalStore(
/* The remote doesn't have a way to access evalStore, so copy
the .drvs. */
RealisedPath::Set drvPaths2;
for (auto & i : paths)
if (auto p = std::get_if<DerivedPath::Built>(&i))
drvPaths2.insert(p->drvPath);
for (const auto & i : paths) {
std::visit(overloaded {
[&](const DerivedPath::Opaque & bp) {
// Do nothing, path is hopefully there already
},
[&](const DerivedPath::Built & bp) {
drvPaths2.insert(bp.drvPath->getBaseStorePath());
},
}, i.raw());
}
copyClosure(*evalStore, *this, drvPaths2);
}
}
@ -757,7 +752,8 @@ std::vector<KeyedBuildResult> RemoteStore::buildPathsWithResults(
};
OutputPathMap outputs;
auto drv = evalStore->readDerivation(bfd.drvPath);
auto drvPath = resolveDerivedPath(*evalStore, *bfd.drvPath);
auto drv = evalStore->readDerivation(drvPath);
const auto outputHashes = staticOutputHashes(*evalStore, drv); // FIXME: expensive
auto built = resolveDerivedPath(*this, bfd, &*evalStore);
for (auto & [output, outputPath] : built) {
@ -765,7 +761,7 @@ std::vector<KeyedBuildResult> RemoteStore::buildPathsWithResults(
if (!outputHash)
throw Error(
"the derivation '%s' doesn't have an output named '%s'",
printStorePath(bfd.drvPath), output);
printStorePath(drvPath), output);
auto outputId = DrvOutput{ *outputHash, output };
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
auto realisation =
@ -837,15 +833,6 @@ void RemoteStore::addTempRoot(const StorePath & path)
}
void RemoteStore::addIndirectRoot(const Path & path)
{
auto conn(getConnection());
conn->to << WorkerProto::Op::AddIndirectRoot << path;
conn.processStderr();
readInt(conn->from);
}
Roots RemoteStore::findRoots(bool censor)
{
auto conn(getConnection());
@ -1090,7 +1077,7 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
return nullptr;
}
void ConnectionHandle::withFramedSink(std::function<void(Sink & sink)> fun)
void RemoteStore::ConnectionHandle::withFramedSink(std::function<void(Sink & sink)> fun)
{
(*this)->to.flush();

8
src/libstore/remote-store.hh
View file

@ -17,7 +17,6 @@ class Pid;
struct FdSink;
struct FdSource;
template<typename T> class Pool;
struct ConnectionHandle;
struct RemoteStoreConfig : virtual StoreConfig
{
@ -63,7 +62,7 @@ public:
StorePathSet queryDerivationOutputs(const StorePath & path) override;
std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override;
std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore = nullptr) override;
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
StorePathSet querySubstitutablePaths(const StorePathSet & paths) override;
@ -127,8 +126,6 @@ public:
void addTempRoot(const StorePath & path) override;
void addIndirectRoot(const Path & path) override;
Roots findRoots(bool censor) override;
void collectGarbage(const GCOptions & options, GCResults & results) override;
@ -182,6 +179,8 @@ protected:
void setOptions() override;
struct ConnectionHandle;
ConnectionHandle getConnection();
friend struct ConnectionHandle;
@ -199,5 +198,4 @@ private:
std::shared_ptr<Store> evalStore);
};
}

12
src/libstore/ssh-store.cc
View file

@ -1,5 +1,6 @@
#include "ssh-store-config.hh"
#include "store-api.hh"
#include "local-fs-store.hh"
#include "remote-store.hh"
#include "remote-store-connection.hh"
#include "remote-fs-accessor.hh"
@ -61,7 +62,7 @@ public:
std::optional<std::string> getBuildLogExact(const StorePath & path) override
{ unsupported("getBuildLogExact"); }
private:
protected:
struct Connection : RemoteStore::Connection
{
@ -93,9 +94,12 @@ private:
ref<RemoteStore::Connection> SSHStore::openConnection()
{
auto conn = make_ref<Connection>();
conn->sshConn = master.startCommand(
fmt("%s --stdio", remoteProgram)
+ (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get())));
std::string command = remoteProgram + " --stdio";
if (remoteStore.get() != "")
command += " --store " + shellEscape(remoteStore.get());
conn->sshConn = master.startCommand(command);
conn->to = FdSink(conn->sshConn->in.get());
conn->from = FdSource(conn->sshConn->out.get());
return conn;

5
src/libstore/ssh.cc
View file

@ -42,7 +42,10 @@ void SSHMaster::addCommonSSHOpts(Strings & args)
}
bool SSHMaster::isMasterRunning() {
auto res = runProgram(RunOptions {.program = "ssh", .args = {"-O", "check", host}, .mergeStderrToStdout = true});
Strings args = {"-O", "check", host};
addCommonSSHOpts(args);
auto res = runProgram(RunOptions {.program = "ssh", .args = args, .mergeStderrToStdout = true});
return res.first == 0;
}

53
src/libstore/store-api.cc
View file

@ -184,15 +184,15 @@ static std::string makeType(
StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const
{
if (info.hash.hash.type == htSHA256 && info.hash.method == FileIngestionMethod::Recursive) {
return makeStorePath(makeType(*this, "source", info.references), info.hash.hash, name);
if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) {
return makeStorePath(makeType(*this, "source", info.references), info.hash, name);
} else {
assert(info.references.size() == 0);
return makeStorePath("output:out",
hashString(htSHA256,
"fixed:out:"
+ makeFileIngestionPrefix(info.hash.method)
+ info.hash.hash.to_string(Base16, true) + ":"),
+ makeFileIngestionPrefix(info.method)
+ info.hash.to_string(Base16, true) + ":"),
name);
}
}
@ -200,13 +200,13 @@ StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInf
StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const
{
assert(info.hash.hash.type == htSHA256);
assert(info.hash.type == htSHA256);
return makeStorePath(
makeType(*this, "text", StoreReferences {
.others = info.references,
.self = false,
}),
info.hash.hash,
info.hash,
name);
}
@ -232,10 +232,8 @@ std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name,
? hashPath(hashAlgo, srcPath, filter).first
: hashFile(hashAlgo, srcPath);
FixedOutputInfo caInfo {
.hash = {
.method = method,
.hash = h,
},
.references = {},
};
return std::make_pair(makeFixedOutputPath(name, caInfo), h);
@ -248,8 +246,8 @@ StorePath Store::computeStorePathForText(
const StorePathSet & references) const
{
return makeTextPath(name, TextInfo {
{ .hash = hashString(htSHA256, s) },
references,
.hash = hashString(htSHA256, s),
.references = references,
});
}
@ -441,10 +439,8 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath,
*this,
name,
FixedOutputInfo {
.hash = {
.method = method,
.hash = hash,
},
.references = {},
},
narHash,
@ -496,7 +492,7 @@ bool Store::PathInfoCacheValue::isKnownNow()
return std::chrono::steady_clock::now() < time_point + ttl;
}
std::map<std::string, std::optional<StorePath>> Store::queryPartialDerivationOutputMap(const StorePath & path)
std::map<std::string, std::optional<StorePath>> Store::queryStaticPartialDerivationOutputMap(const StorePath & path)
{
std::map<std::string, std::optional<StorePath>> outputs;
auto drv = readInvalidDerivation(path);
@ -506,12 +502,40 @@ std::map<std::string, std::optional<StorePath>> Store::queryPartialDerivationOut
return outputs;
}
std::map<std::string, std::optional<StorePath>> Store::queryPartialDerivationOutputMap(
const StorePath & path,
Store * evalStore_)
{
auto & evalStore = evalStore_ ? *evalStore_ : *this;
auto outputs = evalStore.queryStaticPartialDerivationOutputMap(path);
if (!experimentalFeatureSettings.isEnabled(Xp::CaDerivations))
return outputs;
auto drv = evalStore.readInvalidDerivation(path);
auto drvHashes = staticOutputHashes(*this, drv);
for (auto & [outputName, hash] : drvHashes) {
auto realisation = queryRealisation(DrvOutput{hash, outputName});
if (realisation) {
outputs.insert_or_assign(outputName, realisation->outPath);
} else {
// queryStaticPartialDerivationOutputMap is not guaranteed
// to return std::nullopt for outputs which are not
// statically known.
outputs.insert({outputName, std::nullopt});
}
}
return outputs;
}
OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) {
auto resp = queryPartialDerivationOutputMap(path);
OutputPathMap result;
for (auto & [outName, optOutPath] : resp) {
if (!optOutPath)
throw Error("output '%s' of derivation '%s' has no store path mapped to it", outName, printStorePath(path));
throw MissingRealisation(printStorePath(path), outName);
result.insert_or_assign(outName, *optOutPath);
}
return result;
@ -1472,6 +1496,7 @@ ref<Store> openStore(const std::string & uri_,
if (implem.uriSchemes.count(parsedUri.scheme)) {
auto store = implem.create(parsedUri.scheme, baseURI, params);
if (store) {
experimentalFeatureSettings.require(store->experimentalFeature());
store->init();
store->warnUnknownSettings();
return ref<Store>(store);

37
src/libstore/store-api.hh
View file

@ -99,6 +99,8 @@ typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap;
struct StoreConfig : public Config
{
typedef std::map<std::string, std::string> Params;
using Config::Config;
StoreConfig() = delete;
@ -107,13 +109,28 @@ struct StoreConfig : public Config
virtual ~StoreConfig() { }
/**
* The name of this type of store.
*/
virtual const std::string name() = 0;
/**
* Documentation for this type of store.
*/
virtual std::string doc()
{
return "";
}
/**
* An experimental feature this type store is gated, if it is to be
* experimental.
*/
virtual std::optional<ExperimentalFeature> experimentalFeature() const
{
return std::nullopt;
}
const PathSetting storeDir_{this, settings.nixStore,
"store",
R"(
@ -153,10 +170,6 @@ struct StoreConfig : public Config
class Store : public std::enable_shared_from_this<Store>, public virtual StoreConfig
{
public:
typedef std::map<std::string, std::string> Params;
protected:
struct PathInfoCacheValue {
@ -425,7 +438,20 @@ public:
* derivation. All outputs are mentioned so ones mising the mapping
* are mapped to `std::nullopt`.
*/
virtual std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path);
virtual std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(
const StorePath & path,
Store * evalStore = nullptr);
/**
* Like `queryPartialDerivationOutputMap` but only considers
* statically known output paths (i.e. those that can be gotten from
* the derivation itself.
*
* Just a helper function for implementing
* `queryPartialDerivationOutputMap`.
*/
virtual std::map<std::string, std::optional<StorePath>> queryStaticPartialDerivationOutputMap(
const StorePath & path);
/**
* Query the mapping outputName=>outputPath for the given derivation.
@ -919,6 +945,7 @@ void removeTempRoots();
* Resolve the derived path completely, failing if any derivation output
* is unknown.
*/
StorePath resolveDerivedPath(Store &, const SingleDerivedPath &, Store * evalStore = nullptr);
OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * evalStore = nullptr);

6
src/libstore/tests/derivation.cc
View file

@ -81,7 +81,7 @@ TEST_JSON(DerivationTest, caFixedFlat,
"path": "/nix/store/rhcg9h16sqvlbpsa6dqm57sbr2al6nzg-drv-name-output-name"
})",
(DerivationOutput::CAFixed {
.ca = FixedOutputHash {
.ca = {
.method = FileIngestionMethod::Flat,
.hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="),
},
@ -95,7 +95,7 @@ TEST_JSON(DerivationTest, caFixedNAR,
"path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"
})",
(DerivationOutput::CAFixed {
.ca = FixedOutputHash {
.ca = {
.method = FileIngestionMethod::Recursive,
.hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="),
},
@ -109,7 +109,7 @@ TEST_JSON(DynDerivationTest, caFixedText,
"path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name"
})",
(DerivationOutput::CAFixed {
.ca = TextHash {
.ca = {
.hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="),
},
}),

91
src/libstore/tests/derived-path.cc
View file

@ -17,14 +17,34 @@ Gen<DerivedPath::Opaque> Arbitrary<DerivedPath::Opaque>::arbitrary()
});
}
Gen<SingleDerivedPath::Built> Arbitrary<SingleDerivedPath::Built>::arbitrary()
{
return gen::just(SingleDerivedPath::Built {
.drvPath = make_ref<SingleDerivedPath>(*gen::arbitrary<SingleDerivedPath>()),
.output = (*gen::arbitrary<StorePathName>()).name,
});
}
Gen<DerivedPath::Built> Arbitrary<DerivedPath::Built>::arbitrary()
{
return gen::just(DerivedPath::Built {
.drvPath = *gen::arbitrary<StorePath>(),
.drvPath = make_ref<SingleDerivedPath>(*gen::arbitrary<SingleDerivedPath>()),
.outputs = *gen::arbitrary<OutputsSpec>(),
});
}
Gen<SingleDerivedPath> Arbitrary<SingleDerivedPath>::arbitrary()
{
switch (*gen::inRange<uint8_t>(0, std::variant_size_v<SingleDerivedPath::Raw>)) {
case 0:
return gen::just<SingleDerivedPath>(*gen::arbitrary<SingleDerivedPath::Opaque>());
case 1:
return gen::just<SingleDerivedPath>(*gen::arbitrary<SingleDerivedPath::Built>());
default:
assert(false);
}
}
Gen<DerivedPath> Arbitrary<DerivedPath>::arbitrary()
{
switch (*gen::inRange<uint8_t>(0, std::variant_size_v<DerivedPath::Raw>)) {
@ -45,12 +65,69 @@ class DerivedPathTest : public LibStoreTest
{
};
// FIXME: `RC_GTEST_FIXTURE_PROP` isn't calling `SetUpTestSuite` because it is
// no a real fixture.
//
// See https://github.com/emil-e/rapidcheck/blob/master/doc/gtest.md#rc_gtest_fixture_propfixture-name-args
TEST_F(DerivedPathTest, force_init)
{
/**
* Round trip (string <-> data structure) test for
* `DerivedPath::Opaque`.
*/
TEST_F(DerivedPathTest, opaque) {
std::string_view opaque = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x";
auto elem = DerivedPath::parse(*store, opaque);
auto * p = std::get_if<DerivedPath::Opaque>(&elem);
ASSERT_TRUE(p);
ASSERT_EQ(p->path, store->parseStorePath(opaque));
ASSERT_EQ(elem.to_string(*store), opaque);
}
/**
* Round trip (string <-> data structure) test for a simpler
* `DerivedPath::Built`.
*/
TEST_F(DerivedPathTest, built_opaque) {
std::string_view built = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv^bar,foo";
auto elem = DerivedPath::parse(*store, built);
auto * p = std::get_if<DerivedPath::Built>(&elem);
ASSERT_TRUE(p);
ASSERT_EQ(p->outputs, ((OutputsSpec) OutputsSpec::Names { "foo", "bar" }));
ASSERT_EQ(*p->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque {
.path = store->parseStorePath(built.substr(0, 49)),
}));
ASSERT_EQ(elem.to_string(*store), built);
}
/**
* Round trip (string <-> data structure) test for a more complex,
* inductive `DerivedPath::Built`.
*/
TEST_F(DerivedPathTest, built_built) {
/**
* We set these in tests rather than the regular globals so we don't have
* to worry about race conditions if the tests run concurrently.
*/
ExperimentalFeatureSettings mockXpSettings;
mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations");
std::string_view built = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv^foo^bar,baz";
auto elem = DerivedPath::parse(*store, built, mockXpSettings);
auto * p = std::get_if<DerivedPath::Built>(&elem);
ASSERT_TRUE(p);
ASSERT_EQ(p->outputs, ((OutputsSpec) OutputsSpec::Names { "bar", "baz" }));
auto * drvPath = std::get_if<SingleDerivedPath::Built>(&*p->drvPath);
ASSERT_TRUE(drvPath);
ASSERT_EQ(drvPath->output, "foo");
ASSERT_EQ(*drvPath->drvPath, ((SingleDerivedPath) SingleDerivedPath::Opaque {
.path = store->parseStorePath(built.substr(0, 49)),
}));
ASSERT_EQ(elem.to_string(*store), built);
}
/**
* Without the right experimental features enabled, we cannot parse a
* complex inductive derived path.
*/
TEST_F(DerivedPathTest, built_built_xp) {
ASSERT_THROW(
DerivedPath::parse(*store, "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv^foo^bar,baz"),
MissingExperimentalFeature);
}
RC_GTEST_FIXTURE_PROP(

14
src/libstore/tests/derived-path.hh
View file

@ -12,8 +12,18 @@ namespace rc {
using namespace nix;
template<>
struct Arbitrary<DerivedPath::Opaque> {
static Gen<DerivedPath::Opaque> arbitrary();
struct Arbitrary<SingleDerivedPath::Opaque> {
static Gen<SingleDerivedPath::Opaque> arbitrary();
};
template<>
struct Arbitrary<SingleDerivedPath::Built> {
static Gen<SingleDerivedPath::Built> arbitrary();
};
template<>
struct Arbitrary<SingleDerivedPath> {
static Gen<SingleDerivedPath> arbitrary();
};
template<>

16
src/libstore/tests/downstream-placeholder.cc
View file

@ -5,17 +5,24 @@
namespace nix {
TEST(DownstreamPlaceholder, unknownCaOutput) {
/**
* We set these in tests rather than the regular globals so we don't have
* to worry about race conditions if the tests run concurrently.
*/
ExperimentalFeatureSettings mockXpSettings;
mockXpSettings.set("experimental-features", "ca-derivations");
ASSERT_EQ(
DownstreamPlaceholder::unknownCaOutput(
StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" },
"out").render(),
"out",
mockXpSettings).render(),
"/0c6rn30q4frawknapgwq386zq358m8r6msvywcvc89n6m5p2dgbz");
}
TEST(DownstreamPlaceholder, unknownDerivation) {
/**
* We set these in tests rather than the regular globals so we don't have
* to worry about race conditions if the tests run concurrently.
* Same reason as above
*/
ExperimentalFeatureSettings mockXpSettings;
mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations");
@ -24,7 +31,8 @@ TEST(DownstreamPlaceholder, unknownDerivation) {
DownstreamPlaceholder::unknownDerivation(
DownstreamPlaceholder::unknownCaOutput(
StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv.drv" },
"out"),
"out",
mockXpSettings),
"out",
mockXpSettings).render(),
"/0gn6agqxjyyalf0dpihgyf49xq5hqxgw100f0wydnj6yqrhqsb3w");

Some files were not shown because too many files have changed in this diff Show more