Merge remote-tracking branch 'nixos/master'

This commit is contained in:
Max Headroom 2023-10-31 00:05:37 +01:00
commit e5a5fbc0fb
194 changed files with 3847 additions and 1993 deletions

View file

@ -101,6 +101,9 @@ jobs:
docker_push_image: docker_push_image:
needs: [check_secrets, tests] needs: [check_secrets, tests]
permissions:
contents: read
packages: write
if: >- if: >-
github.event_name == 'push' && github.event_name == 'push' &&
github.ref_name == 'master' && github.ref_name == 'master' &&
@ -126,6 +129,9 @@ jobs:
- run: docker load -i ./result/image.tar.gz - run: docker load -i ./result/image.tar.gz
- run: docker tag nix:$NIX_VERSION nixos/nix:$NIX_VERSION - run: docker tag nix:$NIX_VERSION nixos/nix:$NIX_VERSION
- run: docker tag nix:$NIX_VERSION nixos/nix:master - run: docker tag nix:$NIX_VERSION nixos/nix:master
# We'll deploy the newly built image to both Docker Hub and Github Container Registry.
#
# Push to Docker Hub first
- name: Login to Docker Hub - name: Login to Docker Hub
uses: docker/login-action@v3 uses: docker/login-action@v3
with: with:
@ -133,3 +139,20 @@ jobs:
password: ${{ secrets.DOCKERHUB_TOKEN }} password: ${{ secrets.DOCKERHUB_TOKEN }}
- run: docker push nixos/nix:$NIX_VERSION - run: docker push nixos/nix:$NIX_VERSION
- run: docker push nixos/nix:master - run: docker push nixos/nix:master
# Push to GitHub Container Registry as well
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Push image
run: |
IMAGE_ID=ghcr.io/${{ github.repository_owner }}/nix
# Change all uppercase to lowercase
IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
docker tag nix:$NIX_VERSION $IMAGE_ID:$NIX_VERSION
docker tag nix:$NIX_VERSION $IMAGE_ID:master
docker push $IMAGE_ID:$NIX_VERSION
docker push $IMAGE_ID:master

2
.gitignore vendored
View file

@ -138,7 +138,9 @@ nix-rust/target
result result
# IDE
.vscode/ .vscode/
.idea/
# clangd and possibly more # clangd and possibly more
.cache/ .cache/

View file

@ -27,9 +27,12 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy).
1. Search for related issues that cover what you're going to work on. 1. Search for related issues that cover what you're going to work on.
It could help to mention there that you will work on the issue. It could help to mention there that you will work on the issue.
Issues labeled [good first issue](https://github.com/NixOS/nix/labels/good-first-issue) should be relatively easy to fix and are likely to get merged quickly. Issues labeled [good first issue](https://github.com/NixOS/nix/labels/good%20first%20issue) should be relatively easy to fix and are likely to get merged quickly.
Pull requests addressing issues labeled [idea approved](https://github.com/NixOS/nix/labels/idea%20approved) or [RFC](https://github.com/NixOS/nix/labels/RFC) are especially welcomed by maintainers and will receive prioritised review. Pull requests addressing issues labeled [idea approved](https://github.com/NixOS/nix/labels/idea%20approved) or [RFC](https://github.com/NixOS/nix/labels/RFC) are especially welcomed by maintainers and will receive prioritised review.
If you are proficient with C++, addressing one of the [popular issues](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc) will be highly appreciated by maintainers and Nix users all over the world.
For far-reaching changes, please investigate possible blockers and design implications, and coordinate with maintainers before investing too much time in writing code that may not end up getting merged.
If there is no relevant issue yet and you're not sure whether your change is likely to be accepted, [open an issue](https://github.com/NixOS/nix/issues/new/choose) yourself. If there is no relevant issue yet and you're not sure whether your change is likely to be accepted, [open an issue](https://github.com/NixOS/nix/issues/new/choose) yourself.
2. Check for [pull requests](https://github.com/NixOS/nix/pulls) that might already cover the contribution you are about to make. 2. Check for [pull requests](https://github.com/NixOS/nix/pulls) that might already cover the contribution you are about to make.

View file

@ -2,7 +2,7 @@ let
inherit (builtins) inherit (builtins)
attrNames attrValues fromJSON listToAttrs mapAttrs groupBy attrNames attrValues fromJSON listToAttrs mapAttrs groupBy
concatStringsSep concatMap length lessThan replaceStrings sort; concatStringsSep concatMap length lessThan replaceStrings sort;
inherit (import ./utils.nix) attrsToList concatStrings optionalString filterAttrs trim squash unique; inherit (import <nix/utils.nix>) attrsToList concatStrings optionalString filterAttrs trim squash unique;
showStoreDocs = import ./generate-store-info.nix; showStoreDocs = import ./generate-store-info.nix;
in in

View file

@ -32,7 +32,7 @@ dummy-env = env -i \
NIX_STATE_DIR=/dummy \ NIX_STATE_DIR=/dummy \
NIX_CONFIG='cores = 0' NIX_CONFIG='cores = 0'
nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix=doc/manual --store dummy:// --impure --raw
# re-implement mdBook's include directive to make it usable for terminal output and for proper @docroot@ substitution # re-implement mdBook's include directive to make it usable for terminal output and for proper @docroot@ substitution
define process-includes define process-includes
@ -125,7 +125,7 @@ $(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $
@mv $@.tmp $@ @mv $@.tmp $@
$(d)/xp-features.json: $(bindir)/nix $(d)/xp-features.json: $(bindir)/nix
$(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-xp-features > $@.tmp $(trace-gen) $(dummy-env) $(bindir)/nix __dump-xp-features > $@.tmp
@mv $@.tmp $@ @mv $@.tmp $@
$(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix $(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix
@ -141,7 +141,7 @@ $(d)/src/language/builtin-constants.md: $(d)/language.json $(d)/generate-builtin
@mv $@.tmp $@ @mv $@.tmp $@
$(d)/language.json: $(bindir)/nix $(d)/language.json: $(bindir)/nix
$(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-language > $@.tmp $(trace-gen) $(dummy-env) $(bindir)/nix __dump-language > $@.tmp
@mv $@.tmp $@ @mv $@.tmp $@
# Generate the HTML manual. # Generate the HTML manual.

View file

@ -16,16 +16,8 @@
- [Environment Variables](installation/env-variables.md) - [Environment Variables](installation/env-variables.md)
- [Upgrading Nix](installation/upgrading.md) - [Upgrading Nix](installation/upgrading.md)
- [Uninstalling Nix](installation/uninstall.md) - [Uninstalling Nix](installation/uninstall.md)
- [Package Management](package-management/package-management.md) - [Nix Store](store/index.md)
- [Basic Package Management](package-management/basic-package-mgmt.md) - [File System Object](store/file-system-object.md)
- [Profiles](package-management/profiles.md)
- [Garbage Collection](package-management/garbage-collection.md)
- [Garbage Collector Roots](package-management/garbage-collector-roots.md)
- [Sharing Packages Between Machines](package-management/sharing-packages.md)
- [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md)
- [Copying Closures via SSH](package-management/copy-closure.md)
- [Serving a Nix store via SSH](package-management/ssh-substituter.md)
- [Serving a Nix store via S3](package-management/s3-substituter.md)
- [Nix Language](language/index.md) - [Nix Language](language/index.md)
- [Data Types](language/values.md) - [Data Types](language/values.md)
- [Language Constructs](language/constructs.md) - [Language Constructs](language/constructs.md)
@ -37,7 +29,16 @@
- [Import From Derivation](language/import-from-derivation.md) - [Import From Derivation](language/import-from-derivation.md)
- [Built-in Constants](language/builtin-constants.md) - [Built-in Constants](language/builtin-constants.md)
- [Built-in Functions](language/builtins.md) - [Built-in Functions](language/builtins.md)
- [Package Management](package-management/package-management.md)
- [Profiles](package-management/profiles.md)
- [Garbage Collection](package-management/garbage-collection.md)
- [Garbage Collector Roots](package-management/garbage-collector-roots.md)
- [Advanced Topics](advanced-topics/advanced-topics.md) - [Advanced Topics](advanced-topics/advanced-topics.md)
- [Sharing Packages Between Machines](package-management/sharing-packages.md)
- [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md)
- [Copying Closures via SSH](package-management/copy-closure.md)
- [Serving a Nix store via SSH](package-management/ssh-substituter.md)
- [Serving a Nix store via S3](package-management/s3-substituter.md)
- [Remote Builds](advanced-topics/distributed-builds.md) - [Remote Builds](advanced-topics/distributed-builds.md)
- [Tuning Cores and Jobs](advanced-topics/cores-vs-jobs.md) - [Tuning Cores and Jobs](advanced-topics/cores-vs-jobs.md)
- [Verifying Build Reproducibility](advanced-topics/diff-hook.md) - [Verifying Build Reproducibility](advanced-topics/diff-hook.md)
@ -99,7 +100,6 @@
- [Channels](command-ref/files/channels.md) - [Channels](command-ref/files/channels.md)
- [Default Nix expression](command-ref/files/default-nix-expression.md) - [Default Nix expression](command-ref/files/default-nix-expression.md)
- [Architecture and Design](architecture/architecture.md) - [Architecture and Design](architecture/architecture.md)
- [File System Object](architecture/file-system-object.md)
- [Protocols](protocols/protocols.md) - [Protocols](protocols/protocols.md)
- [Serving Tarball Flakes](protocols/tarball-fetcher.md) - [Serving Tarball Flakes](protocols/tarball-fetcher.md)
- [Derivation "ATerm" file format](protocols/derivation-aterm.md) - [Derivation "ATerm" file format](protocols/derivation-aterm.md)

View file

@ -12,14 +12,14 @@ machine is accessible via SSH and that it has Nix installed. You can
test whether connecting to the remote Nix instance works, e.g. test whether connecting to the remote Nix instance works, e.g.
```console ```console
$ nix store ping --store ssh://mac $ nix store info --store ssh://mac
``` ```
will try to connect to the machine named `mac`. It is possible to will try to connect to the machine named `mac`. It is possible to
specify an SSH identity file as part of the remote store URI, e.g. specify an SSH identity file as part of the remote store URI, e.g.
```console ```console
$ nix store ping --store ssh://mac?ssh-key=/home/alice/my-key $ nix store info --store ssh://mac?ssh-key=/home/alice/my-key
``` ```
Since builds should be non-interactive, the key should not have a Since builds should be non-interactive, the key should not have a

View file

@ -17,9 +17,8 @@ the build loop.
# Prerequisites # Prerequisites
This tutorial assumes you have [configured an S3-compatible binary This tutorial assumes you have configured an [S3-compatible binary cache](@docroot@/command-ref/new-cli/nix3-help-stores.md#s3-binary-cache-store) as a [substituter](../command-ref/conf-file.md#conf-substituters),
cache](../package-management/s3-substituter.md), and that the `root` and that the `root` user's default AWS profile can upload to the bucket.
user's default AWS profile can upload to the bucket.
# Set up a Signing Key # Set up a Signing Key

View file

@ -59,6 +59,7 @@ The [Nix language](../language/index.md) evaluator transforms Nix expressions in
The command line interface and Nix expressions are what users deal with most. The command line interface and Nix expressions are what users deal with most.
> **Note** > **Note**
>
> The Nix language itself does not have a notion of *packages* or *configurations*. > The Nix language itself does not have a notion of *packages* or *configurations*.
> As far as we are concerned here, the inputs and results of a build plan are just data. > As far as we are concerned here, the inputs and results of a build plan are just data.

View file

@ -235,14 +235,14 @@ package like Terraform:
```bash ```bash
#! /usr/bin/env nix-shell #! /usr/bin/env nix-shell
#! nix-shell -i bash --packages "terraform.withPlugins (plugins: [ plugins.openstack ])" #! nix-shell -i bash --packages 'terraform.withPlugins (plugins: [ plugins.openstack ])'
terraform apply terraform apply
``` ```
> **Note** > **Note**
> >
> You must use double quotes (`"`) when passing a simple Nix expression > You must use single or double quotes (`'`, `"`) when passing a simple Nix expression
> in a nix-shell shebang. > in a nix-shell shebang.
Finally, using the merging of multiple nix-shell shebangs the following Finally, using the merging of multiple nix-shell shebangs the following
@ -251,7 +251,7 @@ branch):
```haskell ```haskell
#! /usr/bin/env nix-shell #! /usr/bin/env nix-shell
#! nix-shell -i runghc --packages "haskellPackages.ghcWithPackages (ps: [ps.download-curl ps.tagsoup])" #! nix-shell -i runghc --packages 'haskellPackages.ghcWithPackages (ps: [ps.download-curl ps.tagsoup])'
#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-20.03.tar.gz #! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-20.03.tar.gz
import Network.Curl.Download import Network.Curl.Download

View file

@ -73,6 +73,17 @@ It should therefore aim to be correct, consistent, complete, and easy to navigat
Non-trivial examples may need additional explanation, especially if they use concepts from outside the given context. Non-trivial examples may need additional explanation, especially if they use concepts from outside the given context.
- Always explain code examples in the text.
Use comments in code samples very sparingly, for instance to highlight a particular aspect.
Readers tend to glance over large amounts of code when scanning for information.
Especially beginners will likely find reading more complex-looking code strenuous and may therefore avoid it altogether.
If a code sample appears to require a lot of inline explanation, consider replacing it with a simpler one.
If that's not possible, break the example down into multiple parts, explain them separately, and then show the combined result at the end.
This should be a last resort, as that would amount to writing a [tutorial](https://diataxis.fr/tutorials/) on the given subject.
- Use British English. - Use British English.
This is a somewhat arbitrary choice to force consistency, and accounts for the fact that a majority of Nix users and developers are from Europe. This is a somewhat arbitrary choice to force consistency, and accounts for the fact that a majority of Nix users and developers are from Europe.

View file

@ -58,22 +58,16 @@
- [store]{#gloss-store} - [store]{#gloss-store}
The location in the file system where store objects live. Typically A collection of store objects, with operations to manipulate that collection.
`/nix/store`. See [Nix Store] for details.
From the perspective of the location where Nix is There are many types of stores.
invoked, the Nix store can be referred to See [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md) for a complete list.
as a "_local_" or a "_remote_" one:
+ A [local store]{#gloss-local-store} exists on the filesystem of From the perspective of the location where Nix is invoked, the Nix store can be referred to _local_ or _remote_.
the machine where Nix is invoked. You can use other Only a [local store]{#gloss-local-store} exposes a location in the file system of the machine where Nix is invoked that allows access to store objects, typically `/nix/store`.
local stores by passing the `--store` flag to the Local stores can be used for building [derivations](#derivation).
`nix` command. Local stores can be used for building derivations. See [Local Store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) for details.
+ A *remote store* exists anywhere other than the
local filesystem. One example is the `/nix/store`
directory on another machine, accessed via `ssh` or
served by the `nix-serve` Perl script.
[store]: #gloss-store [store]: #gloss-store
[local store]: #gloss-local-store [local store]: #gloss-local-store
@ -103,15 +97,19 @@
The Nix data model for representing simplified file system data. The Nix data model for representing simplified file system data.
See [File System Object](@docroot@/architecture/file-system-object.md) for details. See [File System Object](@docroot@/store/file-system-object.md) for details.
[file system object]: #gloss-file-system-object [file system object]: #gloss-file-system-object
- [store object]{#gloss-store-object} - [store object]{#gloss-store-object}
A store object consists of a [file system object], [reference]s to other store objects, and other metadata. Part of the contents of a [store].
A store object consists of a [file system object], [references][reference] to other store objects, and other metadata.
It can be referred to by a [store path]. It can be referred to by a [store path].
See [Store Object](@docroot@/store/index.md#store-object) for details.
[store object]: #gloss-store-object [store object]: #gloss-store-object
- [IFD]{#gloss-ifd} - [IFD]{#gloss-ifd}
@ -207,6 +205,7 @@
- [output]{#gloss-output} - [output]{#gloss-output}
A [store object] produced by a [derivation]. A [store object] produced by a [derivation].
See [the `outputs` argument to the `derivation` function](@docroot@/language/derivations.md#attr-outputs) for details.
[output]: #gloss-output [output]: #gloss-output

View file

@ -3,14 +3,14 @@
To run the latest stable release of Nix with Docker run the following command: To run the latest stable release of Nix with Docker run the following command:
```console ```console
$ docker run -ti nixos/nix $ docker run -ti ghcr.io/nixos/nix
Unable to find image 'nixos/nix:latest' locally Unable to find image 'ghcr.io/nixos/nix:latest' locally
latest: Pulling from nixos/nix latest: Pulling from ghcr.io/nixos/nix
5843afab3874: Pull complete 5843afab3874: Pull complete
b52bf13f109c: Pull complete b52bf13f109c: Pull complete
1e2415612aa3: Pull complete 1e2415612aa3: Pull complete
Digest: sha256:27f6e7f60227e959ee7ece361f75d4844a40e1cc6878b6868fe30140420031ff Digest: sha256:27f6e7f60227e959ee7ece361f75d4844a40e1cc6878b6868fe30140420031ff
Status: Downloaded newer image for nixos/nix:latest Status: Downloaded newer image for ghcr.io/nixos/nix:latest
35ca4ada6e96:/# nix --version 35ca4ada6e96:/# nix --version
nix (Nix) 2.3.12 nix (Nix) 2.3.12
35ca4ada6e96:/# exit 35ca4ada6e96:/# exit

View file

@ -8,8 +8,6 @@ It outputs an attribute set, and produces a [store derivation] as a side effect
[store derivation]: @docroot@/glossary.md#gloss-store-derivation [store derivation]: @docroot@/glossary.md#gloss-store-derivation
<!-- FIXME: add a section on output attributes -->
## Input attributes ## Input attributes
### Required ### Required
@ -17,11 +15,22 @@ It outputs an attribute set, and produces a [store derivation] as a side effect
- [`name`]{#attr-name} ([String](@docroot@/language/values.md#type-string)) - [`name`]{#attr-name} ([String](@docroot@/language/values.md#type-string))
A symbolic name for the derivation. A symbolic name for the derivation.
It is added to the [store derivation]'s [path](@docroot@/glossary.md#gloss-store-path) and its [output paths][output path]. It is added to the [store path] of the corresponding [store derivation] as well as to its [output paths](@docroot@/glossary.md#gloss-output-path).
Example: `name = "hello";` [store path]: @docroot@/glossary.md#gloss-store-path
> **Example**
>
> ```nix
> derivation {
> name = "hello";
> # ...
> }
> ```
>
> The store derivation's path will be `/nix/store/<hash>-hello.drv`.
> The [output](#attr-outputs) paths will be of the form `/nix/store/<hash>-hello[-<output>]`
The store derivation's path will be `/nix/store/<hash>-hello.drv`, and the output paths will be of the form `/nix/store/<hash>-hello[-<output>]`
- [`system`]{#attr-system} ([String](@docroot@/language/values.md#type-string)) - [`system`]{#attr-system} ([String](@docroot@/language/values.md#type-string))
The system type on which the [`builder`](#attr-builder) executable is meant to be run. The system type on which the [`builder`](#attr-builder) executable is meant to be run.
@ -29,77 +38,175 @@ It outputs an attribute set, and produces a [store derivation] as a side effect
A necessary condition for Nix to build derivations locally is that the `system` attribute matches the current [`system` configuration option]. A necessary condition for Nix to build derivations locally is that the `system` attribute matches the current [`system` configuration option].
It can automatically [build on other platforms](../advanced-topics/distributed-builds.md) by forwarding build requests to other machines. It can automatically [build on other platforms](../advanced-topics/distributed-builds.md) by forwarding build requests to other machines.
Examples:
`system = "x86_64-linux";`
`system = builtins.currentSystem;`
[`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation.
[`system` configuration option]: @docroot@/command-ref/conf-file.md#conf-system [`system` configuration option]: @docroot@/command-ref/conf-file.md#conf-system
> **Example**
>
> Declare a derivation to be built on a specific system type:
>
> ```nix
> derivation {
> # ...
> system = "x86_64-linux";
> # ...
> }
> ```
> **Example**
>
> Declare a derivation to be built on the system type that evaluates the expression:
>
> ```nix
> derivation {
> # ...
> system = builtins.currentSystem;
> # ...
> }
> ```
>
> [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation.
- [`builder`]{#attr-builder} ([Path](@docroot@/language/values.md#type-path) | [String](@docroot@/language/values.md#type-string)) - [`builder`]{#attr-builder} ([Path](@docroot@/language/values.md#type-path) | [String](@docroot@/language/values.md#type-string))
Path to an executable that will perform the build. Path to an executable that will perform the build.
Examples: > **Example**
>
> Use the file located at `/bin/bash` as the builder executable:
>
> ```nix
> derivation {
> # ...
> builder = "/bin/bash";
> # ...
> };
> ```
`builder = "/bin/bash";` <!-- -->
`builder = ./builder.sh;` > **Example**
>
> Copy a local file to the Nix store for use as the builder executable:
>
> ```nix
> derivation {
> # ...
> builder = ./builder.sh;
> # ...
> };
> ```
`builder = "${pkgs.python}/bin/python";` <!-- -->
> **Example**
>
> Use a file from another derivation as the builder executable:
>
> ```nix
> let pkgs = import <nixpkgs> {}; in
> derivation {
> # ...
> builder = "${pkgs.python}/bin/python";
> # ...
> };
> ```
### Optional ### Optional
- [`args`]{#attr-args} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) Default: `[ ]` - [`args`]{#attr-args} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string))
Default: `[ ]`
Command-line arguments to be passed to the [`builder`](#attr-builder) executable. Command-line arguments to be passed to the [`builder`](#attr-builder) executable.
Example: `args = [ "-c" "echo hello world > $out" ];` > **Example**
>
> Pass arguments to Bash to interpret a shell command:
>
> ```nix
> derivation {
> # ...
> builder = "/bin/bash";
> args = [ "-c" "echo hello world > $out" ];
> # ...
> };
> ```
- [`outputs`]{#attr-outputs} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) Default: `[ "out" ]` - [`outputs`]{#attr-outputs} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string))
Default: `[ "out" ]`
Symbolic outputs of the derivation. Symbolic outputs of the derivation.
Each output name is passed to the [`builder`](#attr-builder) executable as an environment variable with its value set to the corresponding [output path]. Each output name is passed to the [`builder`](#attr-builder) executable as an environment variable with its value set to the corresponding [store path].
[output path]: @docroot@/glossary.md#gloss-output-path By default, a derivation produces a single output called `out`.
However, derivations can produce multiple outputs.
By default, a derivation produces a single output path called `out`.
However, derivations can produce multiple output paths.
This allows the associated [store objects](@docroot@/glossary.md#gloss-store-object) and their [closures](@docroot@/glossary.md#gloss-closure) to be copied or garbage-collected separately. This allows the associated [store objects](@docroot@/glossary.md#gloss-store-object) and their [closures](@docroot@/glossary.md#gloss-closure) to be copied or garbage-collected separately.
Examples: > **Example**
>
> Imagine a library package that provides a dynamic library, header files, and documentation.
> A program that links against such a library doesnt need the header files and documentation at runtime, and it doesnt need the documentation at build time.
> Thus, the library package could specify:
>
> ```nix
> derivation {
> # ...
> outputs = [ "lib" "dev" "doc" ];
> # ...
> }
> ```
>
> This will cause Nix to pass environment variables `lib`, `dev`, and `doc` to the builder containing the intended store paths of each output.
> The builder would typically do something like
>
> ```bash
> ./configure \
> --libdir=$lib/lib \
> --includedir=$dev/include \
> --docdir=$doc/share/doc
> ```
>
> for an Autoconf-style package.
Imagine a library package that provides a dynamic library, header files, and documentation. The name of an output is combined with the name of the derivation to create the name part of the output's store path, unless it is `out`, in which case just the name of the derivation is used.
A program that links against such a library doesnt need the header files and documentation at runtime, and it doesnt need the documentation at build time.
Thus, the library package could specify:
```nix > **Example**
derivation { >
# ... >
outputs = [ "lib" "dev" "doc" ]; > ```nix
# ... > derivation {
} > name = "example";
``` > outputs = [ "lib" "dev" "doc" "out" ];
> # ...
> }
> ```
>
> The store derivation path will be `/nix/store/<hash>-example.drv`.
> The output paths will be
> - `/nix/store/<hash>-example-lib`
> - `/nix/store/<hash>-example-dev`
> - `/nix/store/<hash>-example-doc`
> - `/nix/store/<hash>-example`
This will cause Nix to pass environment variables `lib`, `dev`, and `doc` to the builder containing the intended store paths of each output. You can refer to each output of a derivation by selecting it as an attribute.
The builder would typically do something like The first element of `outputs` determines the *default output* and ends up at the top-level.
```bash > **Example**
./configure \ >
--libdir=$lib/lib \ > Select an output by attribute name:
--includedir=$dev/include \ >
--docdir=$doc/share/doc > ```nix
``` > let
> myPackage = derivation {
for an Autoconf-style package. > name = "example";
> outputs = [ "lib" "dev" "doc" "out" ];
You can refer to each output of a derivation by selecting it as an attribute, e.g. `myPackage.lib` or `myPackage.doc`. > # ...
> };
The first element of `outputs` determines the *default output*. > in myPackage.dev
Therefore, in the given example, `myPackage` is equivalent to `myPackage.lib`. > ```
>
> Since `lib` is the first output, `myPackage` is equivalent to `myPackage.lib`.
<!-- FIXME: refer to the output attributes when we have one --> <!-- FIXME: refer to the output attributes when we have one -->
@ -123,8 +230,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect
reside in the Nix store. reside in the Nix store.
- A *derivation* causes that derivation to be built prior to the - A *derivation* causes that derivation to be built prior to the
present derivation; its default output path is put in the present derivation. The environment variable is set to the [store path] of the derivation's default [output](#attr-outputs).
environment variable.
- Lists of the previous types are also allowed. They are simply - Lists of the previous types are also allowed. They are simply
concatenated, separated by spaces. concatenated, separated by spaces.
@ -132,6 +238,8 @@ It outputs an attribute set, and produces a [store derivation] as a side effect
- `true` is passed as the string `1`, `false` and `null` are - `true` is passed as the string `1`, `false` and `null` are
passed as an empty string. passed as an empty string.
<!-- FIXME: add a section on output attributes -->
## Builder execution ## Builder execution
The [`builder`](#attr-builder) is executed as follows: The [`builder`](#attr-builder) is executed as follows:

View file

@ -59,10 +59,14 @@ An attribute path is a dot-separated list of [attribute names](./values.md#attri
Test whether [attribute set] *attrset* contains the attribute denoted by *attrpath*. Test whether [attribute set] *attrset* contains the attribute denoted by *attrpath*.
The result is a [Boolean] value. The result is a [Boolean] value.
See also: [`builtins.hasAttr`](@docroot@/language/builtins.md#builtins-hasAttr)
[Boolean]: ./values.md#type-boolean [Boolean]: ./values.md#type-boolean
[Has attribute]: #has-attribute [Has attribute]: #has-attribute
After evaluating *attrset* and *attrpath*, the computational complexity is O(log(*n*)) for *n* attributes in the *attrset*
## Arithmetic ## Arithmetic
Numbers are type-compatible: Numbers are type-compatible:

View file

@ -1,179 +0,0 @@
# Basic Package Management
The main command for package management is
[`nix-env`](../command-ref/nix-env.md). You can use it to install,
upgrade, and erase packages, and to query what packages are installed
or are available for installation.
In Nix, different users can have different “views” on the set of
installed applications. That is, there might be lots of applications
present on the system (possibly in many different versions), but users
can have a specific selection of those active — where “active” just
means that it appears in a directory in the users `PATH`. Such a view
on the set of installed applications is called a *user environment*,
which is just a directory tree consisting of symlinks to the files of
the active applications.
Components are installed from a set of *Nix expressions* that tell Nix
how to build those packages, including, if necessary, their
dependencies. There is a collection of Nix expressions called the
Nixpkgs package collection that contains packages ranging from basic
development stuff such as GCC and Glibc, to end-user applications like
Mozilla Firefox. (Nix is however not tied to the Nixpkgs package
collection; you could write your own Nix expressions based on Nixpkgs,
or completely new ones.)
You can manually download the latest version of Nixpkgs from
<https://github.com/NixOS/nixpkgs>. However, its much more
convenient to use the Nixpkgs [*channel*](../command-ref/nix-channel.md), since it makes
it easy to stay up to date with new versions of Nixpkgs. Nixpkgs is
automatically added to your list of “subscribed” channels when you
install Nix. If this is not the case for some reason, you can add it
as follows:
```console
$ nix-channel --add https://nixos.org/channels/nixpkgs-unstable
$ nix-channel --update
```
> **Note**
>
> On NixOS, youre automatically subscribed to a NixOS channel
> corresponding to your NixOS major release (e.g.
> <http://nixos.org/channels/nixos-21.11>). A NixOS channel is identical
> to the Nixpkgs channel, except that it contains only Linux binaries
> and is updated only if a set of regression tests succeed.
You can view the set of available packages in Nixpkgs:
```console
$ nix-env --query --available --attr-path
nixpkgs.aterm aterm-2.2
nixpkgs.bash bash-3.0
nixpkgs.binutils binutils-2.15
nixpkgs.bison bison-1.875d
nixpkgs.blackdown blackdown-1.4.2
nixpkgs.bzip2 bzip2-1.0.2
```
The flag `-q` specifies a query operation, `-a` means that you want
to show the “available” (i.e., installable) packages, as opposed to the
installed packages, and `-P` prints the attribute paths that can be used
to unambiguously select a package for installation (listed in the first column).
If you downloaded Nixpkgs yourself, or if you checked it out from GitHub,
then you need to pass the path to your Nixpkgs tree using the `-f` flag:
```console
$ nix-env --query --available --attr-path --file /path/to/nixpkgs
aterm aterm-2.2
bash bash-3.0
```
where */path/to/nixpkgs* is where youve unpacked or checked out
Nixpkgs.
You can filter the packages by name:
```console
$ nix-env --query --available --attr-path firefox
nixpkgs.firefox-esr firefox-91.3.0esr
nixpkgs.firefox firefox-94.0.1
```
and using regular expressions:
```console
$ nix-env --query --available --attr-path 'firefox.*'
```
It is also possible to see the *status* of available packages, i.e.,
whether they are installed into the user environment and/or present in
the system:
```console
$ nix-env --query --available --attr-path --status
-PS nixpkgs.bash bash-3.0
--S nixpkgs.binutils binutils-2.15
IPS nixpkgs.bison bison-1.875d
```
The first character (`I`) indicates whether the package is installed in
your current user environment. The second (`P`) indicates whether it is
present on your system (in which case installing it into your user
environment would be a very quick operation). The last one (`S`)
indicates whether there is a so-called *substitute* for the package,
which is Nixs mechanism for doing binary deployment. It just means that
Nix knows that it can fetch a pre-built package from somewhere
(typically a network server) instead of building it locally.
You can install a package using `nix-env --install --attr `. For instance,
```console
$ nix-env --install --attr nixpkgs.subversion
```
will install the package called `subversion` from `nixpkgs` channel (which is, of course, the
[Subversion version management system](http://subversion.tigris.org/)).
> **Note**
>
> When you ask Nix to install a package, it will first try to get it in
> pre-compiled form from a *binary cache*. By default, Nix will use the
> binary cache <https://cache.nixos.org>; it contains binaries for most
> packages in Nixpkgs. Only if no binary is available in the binary
> cache, Nix will build the package from source. So if `nix-env
> -iA nixpkgs.subversion` results in Nix building stuff from source, then either
> the package is not built for your platform by the Nixpkgs build
> servers, or your version of Nixpkgs is too old or too new. For
> instance, if you have a very recent checkout of Nixpkgs, then the
> Nixpkgs build servers may not have had a chance to build everything
> and upload the resulting binaries to <https://cache.nixos.org>. The
> Nixpkgs channel is only updated after all binaries have been uploaded
> to the cache, so if you stick to the Nixpkgs channel (rather than
> using a Git checkout of the Nixpkgs tree), you will get binaries for
> most packages.
Naturally, packages can also be uninstalled. Unlike when installing, you will
need to use the derivation name (though the version part can be omitted),
instead of the attribute path, as `nix-env` does not record which attribute
was used for installing:
```console
$ nix-env --uninstall subversion
```
Upgrading to a new version is just as easy. If you have a new release of
Nix Packages, you can do:
```console
$ nix-env --upgrade --attr nixpkgs.subversion
```
This will *only* upgrade Subversion if there is a “newer” version in the
new set of Nix expressions, as defined by some pretty arbitrary rules
regarding ordering of version numbers (which generally do what youd
expect of them). To just unconditionally replace Subversion with
whatever version is in the Nix expressions, use `-i` instead of `-u`;
`-i` will remove whatever version is already installed.
You can also upgrade all packages for which there are newer versions:
```console
$ nix-env --upgrade
```
Sometimes its useful to be able to ask what `nix-env` would do, without
actually doing it. For instance, to find out what packages would be
upgraded by `nix-env --upgrade `, you can do
```console
$ nix-env --upgrade --dry-run
(dry run; not doing anything)
upgrading `libxslt-1.1.0' to `libxslt-1.1.10'
upgrading `graphviz-1.10' to `graphviz-1.12'
upgrading `coreutils-5.0' to `coreutils-5.2.1'
```

View file

@ -1,115 +0,0 @@
# Serving a Nix store via S3
Nix has [built-in support](@docroot@/command-ref/new-cli/nix3-help-stores.md#s3-binary-cache-store)
for storing and fetching store paths from
Amazon S3 and S3-compatible services. This uses the same *binary*
cache mechanism that Nix usually uses to fetch prebuilt binaries from
[cache.nixos.org](https://cache.nixos.org/).
In this example we will use the bucket named `example-nix-cache`.
## Anonymous Reads to your S3-compatible binary cache
If your binary cache is publicly accessible and does not require
authentication, the simplest and easiest way to use Nix with your S3
compatible binary cache is to use the HTTP URL for that cache.
For AWS S3 the binary cache URL for example bucket will be exactly
<https://example-nix-cache.s3.amazonaws.com> or
<s3://example-nix-cache>. For S3 compatible binary caches, consult that
cache's documentation.
Your bucket will need the following bucket policy:
```json
{
"Id": "DirectReads",
"Version": "2012-10-17",
"Statement": [
{
"Sid": "AllowDirectReads",
"Action": [
"s3:GetObject",
"s3:GetBucketLocation"
],
"Effect": "Allow",
"Resource": [
"arn:aws:s3:::example-nix-cache",
"arn:aws:s3:::example-nix-cache/*"
],
"Principal": "*"
}
]
}
```
## Authenticated Reads to your S3 binary cache
For AWS S3 the binary cache URL for example bucket will be exactly
<s3://example-nix-cache>.
Nix will use the [default credential provider
chain](https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html)
for authenticating requests to Amazon S3.
Nix supports authenticated reads from Amazon S3 and S3 compatible binary
caches.
Your bucket will need a bucket policy allowing the desired users to
perform the `s3:GetObject` and `s3:GetBucketLocation` action on all
objects in the bucket. The [anonymous policy given
above](#anonymous-reads-to-your-s3-compatible-binary-cache) can be
updated to have a restricted `Principal` to support this.
## Authenticated Writes to your S3-compatible binary cache
Nix support fully supports writing to Amazon S3 and S3 compatible
buckets. The binary cache URL for our example bucket will be
<s3://example-nix-cache>.
Nix will use the [default credential provider
chain](https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html)
for authenticating requests to Amazon S3.
Your account will need the following IAM policy to upload to the cache:
```json
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "UploadToCache",
"Effect": "Allow",
"Action": [
"s3:AbortMultipartUpload",
"s3:GetBucketLocation",
"s3:GetObject",
"s3:ListBucket",
"s3:ListBucketMultipartUploads",
"s3:ListMultipartUploadParts",
"s3:PutObject"
],
"Resource": [
"arn:aws:s3:::example-nix-cache",
"arn:aws:s3:::example-nix-cache/*"
]
}
]
}
```
## Examples
To upload with a specific credential profile for Amazon S3:
```console
$ nix copy nixpkgs.hello \
--to 's3://example-nix-cache?profile=cache-upload&region=eu-west-2'
```
To upload to an S3-compatible binary cache:
```console
$ nix copy nixpkgs.hello --to \
's3://example-nix-cache?profile=cache-upload&scheme=https&endpoint=minio.example.com'
```

View file

@ -7,3 +7,11 @@
- The experimental feature `repl-flake` is no longer needed, as its functionality is now part of the `flakes` experimental feature. To get the previous behavior, use the `--file/--expr` flags accordingly. - The experimental feature `repl-flake` is no longer needed, as its functionality is now part of the `flakes` experimental feature. To get the previous behavior, use the `--file/--expr` flags accordingly.
- Introduce new flake installable syntax `flakeref#.attrPath` where the "." prefix denotes no searching of default attribute prefixes like `packages.<SYSTEM>` or `legacyPackages.<SYSTEM>`. - Introduce new flake installable syntax `flakeref#.attrPath` where the "." prefix denotes no searching of default attribute prefixes like `packages.<SYSTEM>` or `legacyPackages.<SYSTEM>`.
- Nix adds `apple-virt` to the default system features on macOS systems that support virtualization. This is similar to what's done for the `kvm` system feature on Linux hosts.
- Introduce a new built-in function [`builtins.convertHash`](@docroot@/language/builtins.md#builtins-convertHash).
- `nix-shell` shebang lines now support single-quoted arguments.
- `builtins.fetchTree` is now marked as stable.

View file

@ -0,0 +1,4 @@
# Nix Store
The *Nix store* is an abstraction used by Nix to store immutable filesystem artifacts (such as software packages) that can have dependencies (*references*) between them.
There are multiple implementations of the Nix store, such as the actual filesystem (`/nix/store`) and binary caches.

View file

@ -474,47 +474,13 @@
hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
passthru.perl-bindings = with final; perl.pkgs.toPerlModule (currentStdenv.mkDerivation { passthru.perl-bindings = final.callPackage ./perl {
name = "nix-super-perl-${version}"; inherit fileset;
stdenv = currentStdenv;
src = fileset.toSource {
root = ./.;
fileset = fileset.intersect baseFiles (fileset.unions [
./perl
./.version
./m4
./mk
]);
}; };
nativeBuildInputs =
[ buildPackages.autoconf-archive
buildPackages.autoreconfHook
buildPackages.pkg-config
];
buildInputs =
[ nix
curl
bzip2
xz
pkgs.perl
boost
]
++ lib.optional (currentStdenv.isLinux || currentStdenv.isDarwin) libsodium
++ lib.optional currentStdenv.isDarwin darwin.apple_sdk.frameworks.Security;
configureFlags = [
"--with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix}"
"--with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix}"
];
enableParallelBuilding = true;
postUnpack = "sourceRoot=$sourceRoot/perl";
});
meta.platforms = lib.platforms.unix; meta.platforms = lib.platforms.unix;
meta.mainProgram = "nix";
}); });
lowdown-nix = with final; currentStdenv.mkDerivation rec { lowdown-nix = with final; currentStdenv.mkDerivation rec {

View file

@ -50,7 +50,9 @@ The team meets twice a week:
1. Code review on pull requests from [In review](#in-review). 1. Code review on pull requests from [In review](#in-review).
2. Other chores and tasks. 2. Other chores and tasks.
Meeting notes are collected on a [collaborative scratchpad](https://pad.lassul.us/Cv7FpYx-Ri-4VjUykQOLAw), and published on Discourse under the [Nix category](https://discourse.nixos.org/c/dev/nix/50). Meeting notes are collected on a [collaborative scratchpad](https://pad.lassul.us/Cv7FpYx-Ri-4VjUykQOLAw).
Notes on issues and pull requests are posted as comments and linked from the meeting notes, so they are easy to find from both places.
[All meeting notes](https://discourse.nixos.org/search?expanded=true&q=Nix%20team%20meeting%20minutes%20%23%20%23dev%3Anix%20in%3Atitle%20order%3Alatest_topic) are published on Discourse under the [Nix category](https://discourse.nixos.org/c/dev/nix/50).
## Project board protocol ## Project board protocol

51
perl/default.nix Normal file
View file

@ -0,0 +1,51 @@
{ lib, fileset
, stdenv
, perl, perlPackages
, autoconf-archive, autoreconfHook, pkg-config
, nix, curl, bzip2, xz, boost, libsodium, darwin
}:
perl.pkgs.toPerlModule (stdenv.mkDerivation {
name = "nix-perl-${nix.version}";
src = fileset.toSource {
root = ../.;
fileset = fileset.unions [
../.version
../m4
../mk
./MANIFEST
./Makefile
./Makefile.config.in
./configure.ac
./lib
./local.mk
];
};
nativeBuildInputs =
[ autoconf-archive
autoreconfHook
pkg-config
];
buildInputs =
[ nix
curl
bzip2
xz
perl
boost
]
++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security;
configureFlags = [
"--with-dbi=${perlPackages.DBI}/${perl.libPrefix}"
"--with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}"
];
enableParallelBuilding = true;
postUnpack = "sourceRoot=$sourceRoot/perl";
})

View file

@ -78,7 +78,7 @@ SV * queryReferences(char * path)
SV * queryPathHash(char * path) SV * queryPathHash(char * path)
PPCODE: PPCODE:
try { try {
auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base32, true); auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(HashFormat::Base32, true);
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
} catch (Error & e) { } catch (Error & e) {
croak("%s", e.what()); croak("%s", e.what());
@ -104,7 +104,7 @@ SV * queryPathInfo(char * path, int base32)
XPUSHs(&PL_sv_undef); XPUSHs(&PL_sv_undef);
else else
XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0))); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0)));
auto s = info->narHash.to_string(base32 ? Base32 : Base16, true); auto s = info->narHash.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, true);
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
mXPUSHi(info->registrationTime); mXPUSHi(info->registrationTime);
mXPUSHi(info->narSize); mXPUSHi(info->narSize);
@ -206,7 +206,7 @@ SV * hashPath(char * algo, int base32, char * path)
PPCODE: PPCODE:
try { try {
Hash h = hashPath(parseHashType(algo), path).first; Hash h = hashPath(parseHashType(algo), path).first;
auto s = h.to_string(base32 ? Base32 : Base16, false); auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false);
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
} catch (Error & e) { } catch (Error & e) {
croak("%s", e.what()); croak("%s", e.what());
@ -217,7 +217,7 @@ SV * hashFile(char * algo, int base32, char * path)
PPCODE: PPCODE:
try { try {
Hash h = hashFile(parseHashType(algo), path); Hash h = hashFile(parseHashType(algo), path);
auto s = h.to_string(base32 ? Base32 : Base16, false); auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false);
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
} catch (Error & e) { } catch (Error & e) {
croak("%s", e.what()); croak("%s", e.what());
@ -228,7 +228,7 @@ SV * hashString(char * algo, int base32, char * s)
PPCODE: PPCODE:
try { try {
Hash h = hashString(parseHashType(algo), s); Hash h = hashString(parseHashType(algo), s);
auto s = h.to_string(base32 ? Base32 : Base16, false); auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false);
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
} catch (Error & e) { } catch (Error & e) {
croak("%s", e.what()); croak("%s", e.what());
@ -239,7 +239,7 @@ SV * convertHash(char * algo, char * s, int toBase32)
PPCODE: PPCODE:
try { try {
auto h = Hash::parseAny(s, parseHashType(algo)); auto h = Hash::parseAny(s, parseHashType(algo));
auto s = h.to_string(toBase32 ? Base32 : Base16, false); auto s = h.to_string(toBase32 ? HashFormat::Base32 : HashFormat::Base16, false);
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
} catch (Error & e) { } catch (Error & e) {
croak("%s", e.what()); croak("%s", e.what());

View file

@ -35,21 +35,28 @@ struct NixMultiCommand : virtual MultiCommand, virtual Command
// For the overloaded run methods // For the overloaded run methods
#pragma GCC diagnostic ignored "-Woverloaded-virtual" #pragma GCC diagnostic ignored "-Woverloaded-virtual"
/* A command that requires a Nix store. */ /**
* A command that requires a \ref Store "Nix store".
*/
struct StoreCommand : virtual Command struct StoreCommand : virtual Command
{ {
StoreCommand(); StoreCommand();
void run() override; void run() override;
ref<Store> getStore(); ref<Store> getStore();
virtual ref<Store> createStore(); virtual ref<Store> createStore();
/**
* Main entry point, with a `Store` provided
*/
virtual void run(ref<Store>) = 0; virtual void run(ref<Store>) = 0;
private: private:
std::shared_ptr<Store> _store; std::shared_ptr<Store> _store;
}; };
/* A command that copies something between `--from` and `--to` /**
stores. */ * A command that copies something between `--from` and `--to` \ref
* Store stores.
*/
struct CopyCommand : virtual StoreCommand struct CopyCommand : virtual StoreCommand
{ {
std::string srcUri, dstUri; std::string srcUri, dstUri;
@ -61,6 +68,9 @@ struct CopyCommand : virtual StoreCommand
ref<Store> getDstStore(); ref<Store> getDstStore();
}; };
/**
* A command that needs to evaluate Nix language expressions.
*/
struct EvalCommand : virtual StoreCommand, MixEvalArgs struct EvalCommand : virtual StoreCommand, MixEvalArgs
{ {
bool startReplOnEvalErrors = false; bool startReplOnEvalErrors = false;
@ -80,20 +90,26 @@ private:
std::shared_ptr<EvalState> evalState; std::shared_ptr<EvalState> evalState;
}; };
/**
* A mixin class for commands that process flakes, adding a few standard
* flake-related options/flags.
*/
struct MixFlakeOptions : virtual Args, EvalCommand struct MixFlakeOptions : virtual Args, EvalCommand
{ {
flake::LockFlags lockFlags; flake::LockFlags lockFlags;
std::optional<std::string> needsFlakeInputCompletion = {};
MixFlakeOptions(); MixFlakeOptions();
virtual std::vector<std::string> getFlakesForCompletion() /**
* The completion for some of these flags depends on the flake(s) in
* question.
*
* This method should be implemented to gather all flakerefs the
* command is operating with (presumably specified via some other
* arguments) so that the completions for these flags can use them.
*/
virtual std::vector<FlakeRef> getFlakeRefsForCompletion()
{ return {}; } { return {}; }
void completeFlakeInput(std::string_view prefix);
void completionHook() override;
}; };
struct SourceExprCommand : virtual Args, MixFlakeOptions struct SourceExprCommand : virtual Args, MixFlakeOptions
@ -129,15 +145,35 @@ struct SourceExprCommand : virtual Args, MixFlakeOptions
virtual Strings getDefaultFlakeAttrPathPrefixes(); virtual Strings getDefaultFlakeAttrPathPrefixes();
void completeInstallable(std::string_view prefix); /**
* Complete an installable from the given prefix.
*/
void completeInstallable(AddCompletions & completions, std::string_view prefix);
/**
* Convenience wrapper around the underlying function to make setting the
* callback easier.
*/
CompleterClosure getCompleteInstallable();
}; };
/**
* A mixin class for commands that need a read-only flag.
*
* What exactly is "read-only" is unspecified, but it will usually be
* the \ref Store "Nix store".
*/
struct MixReadOnlyOption : virtual Args struct MixReadOnlyOption : virtual Args
{ {
MixReadOnlyOption(); MixReadOnlyOption();
}; };
/* Like InstallablesCommand but the installables are not loaded */ /**
* Like InstallablesCommand but the installables are not loaded.
*
* This is needed by `CmdRepl` which wants to load (and reload) the
* installables itself.
*/
struct RawInstallablesCommand : virtual Args, SourceExprCommand struct RawInstallablesCommand : virtual Args, SourceExprCommand
{ {
RawInstallablesCommand(); RawInstallablesCommand();
@ -146,19 +182,22 @@ struct RawInstallablesCommand : virtual Args, SourceExprCommand
void run(ref<Store> store) override; void run(ref<Store> store) override;
// FIXME make const after CmdRepl's override is fixed up // FIXME make const after `CmdRepl`'s override is fixed up
virtual void applyDefaultInstallables(std::vector<std::string> & rawInstallables); virtual void applyDefaultInstallables(std::vector<std::string> & rawInstallables);
bool readFromStdIn = false; bool readFromStdIn = false;
std::vector<std::string> getFlakesForCompletion() override; std::vector<FlakeRef> getFlakeRefsForCompletion() override;
private: private:
std::vector<std::string> rawInstallables; std::vector<std::string> rawInstallables;
}; };
/* A command that operates on a list of "installables", which can be
store paths, attribute paths, Nix expressions, etc. */ /**
* A command that operates on a list of "installables", which can be
* store paths, attribute paths, Nix expressions, etc.
*/
struct InstallablesCommand : RawInstallablesCommand struct InstallablesCommand : RawInstallablesCommand
{ {
virtual void run(ref<Store> store, Installables && installables) = 0; virtual void run(ref<Store> store, Installables && installables) = 0;
@ -166,7 +205,9 @@ struct InstallablesCommand : RawInstallablesCommand
void run(ref<Store> store, std::vector<std::string> && rawInstallables) override; void run(ref<Store> store, std::vector<std::string> && rawInstallables) override;
}; };
/* A command that operates on exactly one "installable" */ /**
* A command that operates on exactly one "installable".
*/
struct InstallableCommand : virtual Args, SourceExprCommand struct InstallableCommand : virtual Args, SourceExprCommand
{ {
InstallableCommand(); InstallableCommand();
@ -175,10 +216,7 @@ struct InstallableCommand : virtual Args, SourceExprCommand
void run(ref<Store> store) override; void run(ref<Store> store) override;
std::vector<std::string> getFlakesForCompletion() override std::vector<FlakeRef> getFlakeRefsForCompletion() override;
{
return {_installable};
}
private: private:
@ -192,7 +230,12 @@ struct MixOperateOnOptions : virtual Args
MixOperateOnOptions(); MixOperateOnOptions();
}; };
/* A command that operates on zero or more store paths. */ /**
* A command that operates on zero or more extant store paths.
*
* If the argument the user passes is a some sort of recipe for a path
* not yet built, it must be built first.
*/
struct BuiltPathsCommand : InstallablesCommand, virtual MixOperateOnOptions struct BuiltPathsCommand : InstallablesCommand, virtual MixOperateOnOptions
{ {
private: private:
@ -224,7 +267,9 @@ struct StorePathsCommand : public BuiltPathsCommand
void run(ref<Store> store, BuiltPaths && paths) override; void run(ref<Store> store, BuiltPaths && paths) override;
}; };
/* A command that operates on exactly one store path. */ /**
* A command that operates on exactly one store path.
*/
struct StorePathCommand : public StorePathsCommand struct StorePathCommand : public StorePathsCommand
{ {
virtual void run(ref<Store> store, const StorePath & storePath) = 0; virtual void run(ref<Store> store, const StorePath & storePath) = 0;
@ -232,7 +277,9 @@ struct StorePathCommand : public StorePathsCommand
void run(ref<Store> store, StorePaths && storePaths) override; void run(ref<Store> store, StorePaths && storePaths) override;
}; };
/* A helper class for registering commands globally. */ /**
* A helper class for registering \ref Command commands globally.
*/
struct RegisterCommand struct RegisterCommand
{ {
typedef std::map<std::vector<std::string>, std::function<ref<Command>()>> Commands; typedef std::map<std::vector<std::string>, std::function<ref<Command>()>> Commands;
@ -288,7 +335,11 @@ struct MixEnvironment : virtual Args {
MixEnvironment(); MixEnvironment();
/* Modify global environ based on ignoreEnvironment, keep, and unset. It's expected that exec will be called before this class goes out of scope, otherwise environ will become invalid. */ /***
* Modify global environ based on `ignoreEnvironment`, `keep`, and
* `unset`. It's expected that exec will be called before this class
* goes out of scope, otherwise `environ` will become invalid.
*/
void setEnviron(); void setEnviron();
}; };
@ -324,9 +375,10 @@ public:
const std::string selfCommandName; const std::string selfCommandName;
}; };
void completeFlakeRef(ref<Store> store, std::string_view prefix); void completeFlakeRef(AddCompletions & completions, ref<Store> store, std::string_view prefix);
void completeFlakeRefWithFragment( void completeFlakeRefWithFragment(
AddCompletions & completions,
ref<EvalState> evalState, ref<EvalState> evalState,
flake::LockFlags lockFlags, flake::LockFlags lockFlags,
Strings attrPathPrefixes, Strings attrPathPrefixes,

View file

@ -9,6 +9,7 @@
#include "flake/flakeref.hh" #include "flake/flakeref.hh"
#include "store-api.hh" #include "store-api.hh"
#include "command.hh" #include "command.hh"
#include "tarball.hh"
namespace nix { namespace nix {
@ -132,8 +133,8 @@ MixEvalArgs::MixEvalArgs()
if (to.subdir != "") extraAttrs["dir"] = to.subdir; if (to.subdir != "") extraAttrs["dir"] = to.subdir;
fetchers::overrideRegistry(from.input, to.input, extraAttrs); fetchers::overrideRegistry(from.input, to.input, extraAttrs);
}}, }},
.completer = {[&](size_t, std::string_view prefix) { .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
completeFlakeRef(openStore(), prefix); completeFlakeRef(completions, openStore(), prefix);
}} }}
}); });
@ -168,14 +169,14 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s)
{ {
if (EvalSettings::isPseudoUrl(s)) { if (EvalSettings::isPseudoUrl(s)) {
auto storePath = fetchers::downloadTarball( auto storePath = fetchers::downloadTarball(
state.store, EvalSettings::resolvePseudoUrl(s), "source", false).tree.storePath; state.store, EvalSettings::resolvePseudoUrl(s), "source", false).storePath;
return state.rootPath(CanonPath(state.store->toRealPath(storePath))); return state.rootPath(CanonPath(state.store->toRealPath(storePath)));
} }
else if (hasPrefix(s, "flake:")) { else if (hasPrefix(s, "flake:")) {
experimentalFeatureSettings.require(Xp::Flakes); experimentalFeatureSettings.require(Xp::Flakes);
auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false); auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false);
auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first.storePath; auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first;
return state.rootPath(CanonPath(state.store->toRealPath(storePath))); return state.rootPath(CanonPath(state.store->toRealPath(storePath)));
} }

View file

@ -32,15 +32,18 @@ const static std::regex attrPathRegex(
R"((?:[a-zA-Z0-9_"-][a-zA-Z0-9_".,^\*-]*))", R"((?:[a-zA-Z0-9_"-][a-zA-Z0-9_".,^\*-]*))",
std::regex::ECMAScript); std::regex::ECMAScript);
void completeFlakeInputPath( static void completeFlakeInputPath(
AddCompletions & completions,
ref<EvalState> evalState, ref<EvalState> evalState,
const FlakeRef & flakeRef, const std::vector<FlakeRef> & flakeRefs,
std::string_view prefix) std::string_view prefix)
{ {
for (auto & flakeRef : flakeRefs) {
auto flake = flake::getFlake(*evalState, flakeRef, true); auto flake = flake::getFlake(*evalState, flakeRef, true);
for (auto & input : flake.inputs) for (auto & input : flake.inputs)
if (hasPrefix(input.first, prefix)) if (hasPrefix(input.first, prefix))
completions->add(input.first); completions.add(input.first);
}
} }
MixFlakeOptions::MixFlakeOptions() MixFlakeOptions::MixFlakeOptions()
@ -94,8 +97,8 @@ MixFlakeOptions::MixFlakeOptions()
.handler = {[&](std::string s) { .handler = {[&](std::string s) {
lockFlags.inputUpdates.insert(flake::parseInputPath(s)); lockFlags.inputUpdates.insert(flake::parseInputPath(s));
}}, }},
.completer = {[&](size_t, std::string_view prefix) { .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
needsFlakeInputCompletion = {std::string(prefix)}; completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix);
}} }}
}); });
@ -110,11 +113,12 @@ MixFlakeOptions::MixFlakeOptions()
flake::parseInputPath(inputPath), flake::parseInputPath(inputPath),
parseFlakeRef(flakeRef, absPath("."), true)); parseFlakeRef(flakeRef, absPath("."), true));
}}, }},
.completer = {[&](size_t n, std::string_view prefix) { .completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) {
if (n == 0) if (n == 0) {
needsFlakeInputCompletion = {std::string(prefix)}; completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix);
else if (n == 1) } else if (n == 1) {
completeFlakeRef(getEvalState()->store, prefix); completeFlakeRef(completions, getEvalState()->store, prefix);
}
}} }}
}); });
@ -161,30 +165,12 @@ MixFlakeOptions::MixFlakeOptions()
} }
} }
}}, }},
.completer = {[&](size_t, std::string_view prefix) { .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
completeFlakeRef(getEvalState()->store, prefix); completeFlakeRef(completions, getEvalState()->store, prefix);
}} }}
}); });
} }
void MixFlakeOptions::completeFlakeInput(std::string_view prefix)
{
auto evalState = getEvalState();
for (auto & flakeRefS : getFlakesForCompletion()) {
auto flakeRef = parseFlakeRefWithFragment(expandTilde(flakeRefS), absPath(".")).first;
auto flake = flake::getFlake(*evalState, flakeRef, true);
for (auto & input : flake.inputs)
if (hasPrefix(input.first, prefix))
completions->add(input.first);
}
}
void MixFlakeOptions::completionHook()
{
if (auto & prefix = needsFlakeInputCompletion)
completeFlakeInput(*prefix);
}
SourceExprCommand::SourceExprCommand() SourceExprCommand::SourceExprCommand()
{ {
addFlag({ addFlag({
@ -302,11 +288,18 @@ Strings SourceExprCommand::getDefaultFlakeAttrPathPrefixes()
}; };
} }
void SourceExprCommand::completeInstallable(std::string_view prefix) Args::CompleterClosure SourceExprCommand::getCompleteInstallable()
{
return [this](AddCompletions & completions, size_t, std::string_view prefix) {
completeInstallable(completions, prefix);
};
}
void SourceExprCommand::completeInstallable(AddCompletions & completions, std::string_view prefix)
{ {
try { try {
if (file) { if (file) {
completionType = ctAttrs; completions.setType(AddCompletions::Type::Attrs);
evalSettings.pureEval = false; evalSettings.pureEval = false;
auto state = getEvalState(); auto state = getEvalState();
@ -341,14 +334,15 @@ void SourceExprCommand::completeInstallable(std::string_view prefix)
std::string name = state->symbols[i.name]; std::string name = state->symbols[i.name];
if (name.find(searchWord) == 0) { if (name.find(searchWord) == 0) {
if (prefix_ == "") if (prefix_ == "")
completions->add(name); completions.add(name);
else else
completions->add(prefix_ + "." + name); completions.add(prefix_ + "." + name);
} }
} }
} }
} else { } else {
completeFlakeRefWithFragment( completeFlakeRefWithFragment(
completions,
getEvalState(), getEvalState(),
lockFlags, lockFlags,
getDefaultFlakeAttrPathPrefixes(), getDefaultFlakeAttrPathPrefixes(),
@ -361,6 +355,7 @@ void SourceExprCommand::completeInstallable(std::string_view prefix)
} }
void completeFlakeRefWithFragment( void completeFlakeRefWithFragment(
AddCompletions & completions,
ref<EvalState> evalState, ref<EvalState> evalState,
flake::LockFlags lockFlags, flake::LockFlags lockFlags,
Strings attrPathPrefixes, Strings attrPathPrefixes,
@ -373,10 +368,9 @@ void completeFlakeRefWithFragment(
bool isAttrPath = std::regex_match(prefix.begin(), prefix.end(), attrPathRegex); bool isAttrPath = std::regex_match(prefix.begin(), prefix.end(), attrPathRegex);
auto hash = prefix.find('#'); auto hash = prefix.find('#');
if (!isAttrPath && hash == std::string::npos) { if (!isAttrPath && hash == std::string::npos) {
completeFlakeRef(evalState->store, prefix); completeFlakeRef(completions, evalState->store, prefix);
} else { } else {
completions.setType(AddCompletions::Type::Attrs);
completionType = ctAttrs;
auto fragment = auto fragment =
isAttrPath isAttrPath
@ -428,9 +422,9 @@ void completeFlakeRefWithFragment(
/* Strip the attrpath prefix. */ /* Strip the attrpath prefix. */
attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size()); attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size());
if (isAttrPath) if (isAttrPath)
completions->add(concatStringsSep(".", evalState->symbols.resolve(attrPath2))); completions.add(concatStringsSep(".", evalState->symbols.resolve(attrPath2)));
else else
completions->add(flakeRefS + "#" + prefixRoot + concatStringsSep(".", evalState->symbols.resolve(attrPath2))); completions.add(flakeRefS + "#" + prefixRoot + concatStringsSep(".", evalState->symbols.resolve(attrPath2)));
} }
} }
} }
@ -441,7 +435,7 @@ void completeFlakeRefWithFragment(
for (auto & attrPath : defaultFlakeAttrPaths) { for (auto & attrPath : defaultFlakeAttrPaths) {
auto attr = root->findAlongAttrPath(parseAttrPath(*evalState, attrPath)); auto attr = root->findAlongAttrPath(parseAttrPath(*evalState, attrPath));
if (!attr) continue; if (!attr) continue;
completions->add(flakeRefS + "#" + prefixRoot); completions.add(flakeRefS + "#" + prefixRoot);
} }
} }
} }
@ -450,15 +444,15 @@ void completeFlakeRefWithFragment(
} }
} }
void completeFlakeRef(ref<Store> store, std::string_view prefix) void completeFlakeRef(AddCompletions & completions, ref<Store> store, std::string_view prefix)
{ {
if (!experimentalFeatureSettings.isEnabled(Xp::Flakes)) if (!experimentalFeatureSettings.isEnabled(Xp::Flakes))
return; return;
if (prefix == "") if (prefix == "")
completions->add("."); completions.add(".");
completeDir(0, prefix); Args::completeDir(completions, 0, prefix);
/* Look for registry entries that match the prefix. */ /* Look for registry entries that match the prefix. */
for (auto & registry : fetchers::getRegistries(store)) { for (auto & registry : fetchers::getRegistries(store)) {
@ -467,10 +461,10 @@ void completeFlakeRef(ref<Store> store, std::string_view prefix)
if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) { if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) {
std::string from2(from, 6); std::string from2(from, 6);
if (hasPrefix(from2, prefix)) if (hasPrefix(from2, prefix))
completions->add(from2); completions.add(from2);
} else { } else {
if (hasPrefix(from, prefix)) if (hasPrefix(from, prefix))
completions->add(from); completions.add(from);
} }
} }
} }
@ -905,9 +899,7 @@ RawInstallablesCommand::RawInstallablesCommand()
expectArgs({ expectArgs({
.label = "installables", .label = "installables",
.handler = {&rawInstallables}, .handler = {&rawInstallables},
.completer = {[&](size_t, std::string_view prefix) { .completer = getCompleteInstallable(),
completeInstallable(prefix);
}}
}); });
} }
@ -920,6 +912,17 @@ void RawInstallablesCommand::applyDefaultInstallables(std::vector<std::string> &
} }
} }
std::vector<FlakeRef> RawInstallablesCommand::getFlakeRefsForCompletion()
{
applyDefaultInstallables(rawInstallables);
std::vector<FlakeRef> res;
for (auto i : rawInstallables)
res.push_back(parseFlakeRefWithFragment(
expandTilde(i),
absPath(".")).first);
return res;
}
void RawInstallablesCommand::run(ref<Store> store) void RawInstallablesCommand::run(ref<Store> store)
{ {
if (readFromStdIn && !isatty(STDIN_FILENO)) { if (readFromStdIn && !isatty(STDIN_FILENO)) {
@ -933,10 +936,13 @@ void RawInstallablesCommand::run(ref<Store> store)
run(store, std::move(rawInstallables)); run(store, std::move(rawInstallables));
} }
std::vector<std::string> RawInstallablesCommand::getFlakesForCompletion() std::vector<FlakeRef> InstallableCommand::getFlakeRefsForCompletion()
{ {
applyDefaultInstallables(rawInstallables); return {
return rawInstallables; parseFlakeRefWithFragment(
expandTilde(_installable),
absPath(".")).first
};
} }
void InstallablesCommand::run(ref<Store> store, std::vector<std::string> && rawInstallables) void InstallablesCommand::run(ref<Store> store, std::vector<std::string> && rawInstallables)
@ -952,9 +958,7 @@ InstallableCommand::InstallableCommand()
.label = "installable", .label = "installable",
.optional = true, .optional = true,
.handler = {&_installable}, .handler = {&_installable},
.completer = {[&](size_t, std::string_view prefix) { .completer = getCompleteInstallable(),
completeInstallable(prefix);
}}
}); });
} }

View file

@ -132,7 +132,7 @@ std::pair<SourcePath, uint32_t> findPackageFilename(EvalState & state, Value & v
if (colon == std::string::npos) fail(); if (colon == std::string::npos) fail();
std::string filename(fn, 0, colon); std::string filename(fn, 0, colon);
auto lineno = std::stoi(std::string(fn, colon + 1, std::string::npos)); auto lineno = std::stoi(std::string(fn, colon + 1, std::string::npos));
return {CanonPath(fn.substr(0, colon)), lineno}; return {SourcePath{path.accessor, CanonPath(fn.substr(0, colon))}, lineno};
} catch (std::invalid_argument & e) { } catch (std::invalid_argument & e) {
fail(); fail();
abort(); abort();

View file

@ -50,7 +50,7 @@ struct AttrDb
Path cacheDir = getCacheDir() + "/nix/eval-cache-v5"; Path cacheDir = getCacheDir() + "/nix/eval-cache-v5";
createDirs(cacheDir); createDirs(cacheDir);
Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite"; Path dbPath = cacheDir + "/" + fingerprint.to_string(HashFormat::Base16, false) + ".sqlite";
state->db = SQLite(dbPath); state->db = SQLite(dbPath);
state->db.isCache(); state->db.isCache();

View file

@ -12,6 +12,8 @@
#include "function-trace.hh" #include "function-trace.hh"
#include "profiles.hh" #include "profiles.hh"
#include "print.hh" #include "print.hh"
#include "fs-input-accessor.hh"
#include "memory-input-accessor.hh"
#include <algorithm> #include <algorithm>
#include <chrono> #include <chrono>
@ -503,7 +505,17 @@ EvalState::EvalState(
, sOutputSpecified(symbols.create("outputSpecified")) , sOutputSpecified(symbols.create("outputSpecified"))
, repair(NoRepair) , repair(NoRepair)
, emptyBindings(0) , emptyBindings(0)
, derivationInternal(rootPath(CanonPath("/builtin/derivation.nix"))) , rootFS(makeFSInputAccessor(CanonPath::root))
, corepkgsFS(makeMemoryInputAccessor())
, internalFS(makeMemoryInputAccessor())
, derivationInternal{corepkgsFS->addFile(
CanonPath("derivation-internal.nix"),
#include "primops/derivation.nix.gen.hh"
)}
, callFlakeInternal{internalFS->addFile(
CanonPath("call-flake.nix"),
#include "flake/call-flake.nix.gen.hh"
)}
, store(store) , store(store)
, buildStore(buildStore ? buildStore : store) , buildStore(buildStore ? buildStore : store)
, debugRepl(nullptr) , debugRepl(nullptr)
@ -539,7 +551,7 @@ EvalState::EvalState(
auto r = resolveSearchPathPath(i.path); auto r = resolveSearchPathPath(i.path);
if (!r) continue; if (!r) continue;
auto path = *std::move(r); auto path = std::move(*r);
if (store->isInStore(path)) { if (store->isInStore(path)) {
try { try {
@ -555,6 +567,11 @@ EvalState::EvalState(
} }
} }
corepkgsFS->addFile(
CanonPath("fetchurl.nix"),
#include "fetchurl.nix.gen.hh"
);
createBaseEnv(); createBaseEnv();
} }
@ -585,6 +602,9 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value &
SourcePath EvalState::checkSourcePath(const SourcePath & path_) SourcePath EvalState::checkSourcePath(const SourcePath & path_)
{ {
// Don't check non-rootFS accessors, they're in a different namespace.
if (path_.accessor != ref<InputAccessor>(rootFS)) return path_;
if (!allowedPaths) return path_; if (!allowedPaths) return path_;
auto i = resolvedPaths.find(path_.path.abs()); auto i = resolvedPaths.find(path_.path.abs());
@ -599,8 +619,6 @@ SourcePath EvalState::checkSourcePath(const SourcePath & path_)
*/ */
Path abspath = canonPath(path_.path.abs()); Path abspath = canonPath(path_.path.abs());
if (hasPrefix(abspath, corepkgsPrefix)) return CanonPath(abspath);
for (auto & i : *allowedPaths) { for (auto & i : *allowedPaths) {
if (isDirOrInDir(abspath, i)) { if (isDirOrInDir(abspath, i)) {
found = true; found = true;
@ -617,7 +635,7 @@ SourcePath EvalState::checkSourcePath(const SourcePath & path_)
/* Resolve symlinks. */ /* Resolve symlinks. */
debug("checking access to '%s'", abspath); debug("checking access to '%s'", abspath);
SourcePath path = CanonPath(canonPath(abspath, true)); SourcePath path = rootPath(CanonPath(canonPath(abspath, true)));
for (auto & i : *allowedPaths) { for (auto & i : *allowedPaths) {
if (isDirOrInDir(path.path.abs(), i)) { if (isDirOrInDir(path.path.abs(), i)) {
@ -649,12 +667,12 @@ void EvalState::checkURI(const std::string & uri)
/* If the URI is a path, then check it against allowedPaths as /* If the URI is a path, then check it against allowedPaths as
well. */ well. */
if (hasPrefix(uri, "/")) { if (hasPrefix(uri, "/")) {
checkSourcePath(CanonPath(uri)); checkSourcePath(rootPath(CanonPath(uri)));
return; return;
} }
if (hasPrefix(uri, "file://")) { if (hasPrefix(uri, "file://")) {
checkSourcePath(CanonPath(std::string(uri, 7))); checkSourcePath(rootPath(CanonPath(std::string(uri, 7))));
return; return;
} }
@ -950,7 +968,7 @@ void Value::mkStringMove(const char * s, const NixStringContext & context)
void Value::mkPath(const SourcePath & path) void Value::mkPath(const SourcePath & path)
{ {
mkPath(makeImmutableString(path.path.abs())); mkPath(&*path.accessor, makeImmutableString(path.path.abs()));
} }
@ -1035,7 +1053,7 @@ std::string EvalState::mkOutputStringRaw(
/* In practice, this is testing for the case of CA derivations, or /* In practice, this is testing for the case of CA derivations, or
dynamic derivations. */ dynamic derivations. */
return optStaticOutputPath return optStaticOutputPath
? store->printStorePath(*std::move(optStaticOutputPath)) ? store->printStorePath(std::move(*optStaticOutputPath))
/* Downstream we would substitute this for an actual path once /* Downstream we would substitute this for an actual path once
we build the floating CA derivation */ we build the floating CA derivation */
: DownstreamPlaceholder::fromSingleDerivedPathBuilt(b, xpSettings).render(); : DownstreamPlaceholder::fromSingleDerivedPathBuilt(b, xpSettings).render();
@ -1165,24 +1183,6 @@ void EvalState::evalFile(const SourcePath & path_, Value & v, bool mustBeTrivial
if (!e) if (!e)
e = parseExprFromFile(checkSourcePath(resolvedPath)); e = parseExprFromFile(checkSourcePath(resolvedPath));
cacheFile(path, resolvedPath, e, v, mustBeTrivial);
}
void EvalState::resetFileCache()
{
fileEvalCache.clear();
fileParseCache.clear();
}
void EvalState::cacheFile(
const SourcePath & path,
const SourcePath & resolvedPath,
Expr * e,
Value & v,
bool mustBeTrivial)
{
fileParseCache[resolvedPath] = e; fileParseCache[resolvedPath] = e;
try { try {
@ -1211,6 +1211,13 @@ void EvalState::cacheFile(
} }
void EvalState::resetFileCache()
{
fileEvalCache.clear();
fileParseCache.clear();
}
void EvalState::eval(Expr * e, Value & v) void EvalState::eval(Expr * e, Value & v)
{ {
e->eval(*this, baseEnv, v); e->eval(*this, baseEnv, v);
@ -2037,7 +2044,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
else if (firstType == nPath) { else if (firstType == nPath) {
if (!context.empty()) if (!context.empty())
state.error("a string that refers to a store path cannot be appended to a path").atPos(pos).withFrame(env, *this).debugThrow<EvalError>(); state.error("a string that refers to a store path cannot be appended to a path").atPos(pos).withFrame(env, *this).debugThrow<EvalError>();
v.mkPath(CanonPath(canonPath(str()))); v.mkPath(state.rootPath(CanonPath(canonPath(str()))));
} else } else
v.mkStringMove(c_str(), context); v.mkStringMove(c_str(), context);
} }
@ -2236,7 +2243,7 @@ BackedStringView EvalState::coerceToString(
!canonicalizePath && !copyToStore !canonicalizePath && !copyToStore
? // FIXME: hack to preserve path literals that end in a ? // FIXME: hack to preserve path literals that end in a
// slash, as in /foo/${x}. // slash, as in /foo/${x}.
v._path v._path.path
: copyToStore : copyToStore
? store->printStorePath(copyPathToStore(context, v.path())) ? store->printStorePath(copyPathToStore(context, v.path()))
: std::string(v.path().path.abs()); : std::string(v.path().path.abs());
@ -2290,7 +2297,7 @@ BackedStringView EvalState::coerceToString(
&& (!v2->isList() || v2->listSize() != 0)) && (!v2->isList() || v2->listSize() != 0))
result += " "; result += " ";
} }
return std::move(result); return result;
} }
} }
@ -2310,7 +2317,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
auto dstPath = i != srcToStore.end() auto dstPath = i != srcToStore.end()
? i->second ? i->second
: [&]() { : [&]() {
auto dstPath = path.fetchToStore(store, path.baseName(), nullptr, repair); auto dstPath = path.fetchToStore(store, path.baseName(), FileIngestionMethod::Recursive, nullptr, repair);
allowPath(dstPath); allowPath(dstPath);
srcToStore.insert_or_assign(path, dstPath); srcToStore.insert_or_assign(path, dstPath);
printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath)); printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath));
@ -2326,10 +2333,34 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx) SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx)
{ {
try {
forceValue(v, pos);
} catch (Error & e) {
e.addTrace(positions[pos], errorCtx);
throw;
}
/* Handle path values directly, without coercing to a string. */
if (v.type() == nPath)
return v.path();
/* Similarly, handle __toString where the result may be a path
value. */
if (v.type() == nAttrs) {
auto i = v.attrs->find(sToString);
if (i != v.attrs->end()) {
Value v1;
callFunction(*i->value, v, v1, pos);
return coerceToPath(pos, v1, context, errorCtx);
}
}
/* Any other value should be coercable to a string, interpreted
relative to the root filesystem. */
auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned(); auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned();
if (path == "" || path[0] != '/') if (path == "" || path[0] != '/')
error("string '%1%' doesn't represent an absolute path", path).withTrace(pos, errorCtx).debugThrow<EvalError>(); error("string '%1%' doesn't represent an absolute path", path).withTrace(pos, errorCtx).debugThrow<EvalError>();
return CanonPath(path); return rootPath(CanonPath(path));
} }
@ -2429,7 +2460,10 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v
return v1.string_view().compare(v2.string_view()) == 0; return v1.string_view().compare(v2.string_view()) == 0;
case nPath: case nPath:
return strcmp(v1._path, v2._path) == 0; return
// FIXME: compare accessors by their fingerprint.
v1._path.accessor == v2._path.accessor
&& strcmp(v1._path.path, v2._path.path) == 0;
case nNull: case nNull:
return true; return true;

View file

@ -24,6 +24,8 @@ class EvalState;
class StorePath; class StorePath;
struct SingleDerivedPath; struct SingleDerivedPath;
enum RepairFlag : bool; enum RepairFlag : bool;
struct FSInputAccessor;
struct MemoryInputAccessor;
/** /**
@ -211,8 +213,26 @@ public:
Bindings emptyBindings; Bindings emptyBindings;
/**
* The accessor for the root filesystem.
*/
const ref<FSInputAccessor> rootFS;
/**
* The in-memory filesystem for <nix/...> paths.
*/
const ref<MemoryInputAccessor> corepkgsFS;
/**
* In-memory filesystem for internal, non-user-callable Nix
* expressions like call-flake.nix.
*/
const ref<MemoryInputAccessor> internalFS;
const SourcePath derivationInternal; const SourcePath derivationInternal;
const SourcePath callFlakeInternal;
/** /**
* Store used to materialise .drv files. * Store used to materialise .drv files.
*/ */
@ -223,7 +243,6 @@ public:
*/ */
const ref<Store> buildStore; const ref<Store> buildStore;
RootValue vCallFlake = nullptr;
RootValue vImportedDrvToDerivation = nullptr; RootValue vImportedDrvToDerivation = nullptr;
/** /**
@ -405,16 +424,6 @@ public:
*/ */
void evalFile(const SourcePath & path, Value & v, bool mustBeTrivial = false); void evalFile(const SourcePath & path, Value & v, bool mustBeTrivial = false);
/**
* Like `evalFile`, but with an already parsed expression.
*/
void cacheFile(
const SourcePath & path,
const SourcePath & resolvedPath,
Expr * e,
Value & v,
bool mustBeTrivial = false);
void resetFileCache(); void resetFileCache();
/** /**
@ -424,7 +433,7 @@ public:
SourcePath findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos); SourcePath findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos);
/** /**
* Try to resolve a search path value (not the optinal key part) * Try to resolve a search path value (not the optional key part)
* *
* If the specified search path element is a URI, download it. * If the specified search path element is a URI, download it.
* *
@ -829,8 +838,6 @@ struct InvalidPathError : EvalError
#endif #endif
}; };
static const std::string corepkgsPrefix{"/__corepkgs__/"};
template<class ErrorType> template<class ErrorType>
void ErrorBuilder::debugThrow() void ErrorBuilder::debugThrow()
{ {

View file

@ -15,7 +15,7 @@ using namespace flake;
namespace flake { namespace flake {
typedef std::pair<fetchers::Tree, FlakeRef> FetchedFlake; typedef std::pair<StorePath, FlakeRef> FetchedFlake;
typedef std::vector<std::pair<FlakeRef, FetchedFlake>> FlakeCache; typedef std::vector<std::pair<FlakeRef, FetchedFlake>> FlakeCache;
static std::optional<FetchedFlake> lookupInFlakeCache( static std::optional<FetchedFlake> lookupInFlakeCache(
@ -34,7 +34,7 @@ static std::optional<FetchedFlake> lookupInFlakeCache(
return std::nullopt; return std::nullopt;
} }
static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree( static std::tuple<StorePath, FlakeRef, FlakeRef> fetchOrSubstituteTree(
EvalState & state, EvalState & state,
const FlakeRef & originalRef, const FlakeRef & originalRef,
bool allowLookup, bool allowLookup,
@ -61,16 +61,16 @@ static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree(
flakeCache.push_back({originalRef, *fetched}); flakeCache.push_back({originalRef, *fetched});
} }
auto [tree, lockedRef] = *fetched; auto [storePath, lockedRef] = *fetched;
debug("got tree '%s' from '%s'", debug("got tree '%s' from '%s'",
state.store->printStorePath(tree.storePath), lockedRef); state.store->printStorePath(storePath), lockedRef);
state.allowPath(tree.storePath); state.allowPath(storePath);
assert(!originalRef.input.getNarHash() || tree.storePath == originalRef.input.computeStorePath(*state.store)); assert(!originalRef.input.getNarHash() || storePath == originalRef.input.computeStorePath(*state.store));
return {std::move(tree), resolvedRef, lockedRef}; return {std::move(storePath), resolvedRef, lockedRef};
} }
static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos) static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos)
@ -202,30 +202,30 @@ static Flake getFlake(
FlakeCache & flakeCache, FlakeCache & flakeCache,
InputPath lockRootPath) InputPath lockRootPath)
{ {
auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree( auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree(
state, originalRef, allowLookup, flakeCache); state, originalRef, allowLookup, flakeCache);
// Guard against symlink attacks. // Guard against symlink attacks.
auto flakeDir = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir, true); auto flakeDir = canonPath(state.store->toRealPath(storePath) + "/" + lockedRef.subdir, true);
auto flakeFile = canonPath(flakeDir + "/flake.nix", true); auto flakeFile = canonPath(flakeDir + "/flake.nix", true);
if (!isInDir(flakeFile, sourceInfo.actualPath)) if (!isInDir(flakeFile, state.store->toRealPath(storePath)))
throw Error("'flake.nix' file of flake '%s' escapes from '%s'", throw Error("'flake.nix' file of flake '%s' escapes from '%s'",
lockedRef, state.store->printStorePath(sourceInfo.storePath)); lockedRef, state.store->printStorePath(storePath));
Flake flake { Flake flake {
.originalRef = originalRef, .originalRef = originalRef,
.resolvedRef = resolvedRef, .resolvedRef = resolvedRef,
.lockedRef = lockedRef, .lockedRef = lockedRef,
.sourceInfo = std::make_shared<fetchers::Tree>(std::move(sourceInfo)) .storePath = storePath,
}; };
if (!pathExists(flakeFile)) if (!pathExists(flakeFile))
throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir); throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir);
Value vInfo; Value vInfo;
state.evalFile(CanonPath(flakeFile), vInfo, false); // FIXME: symlink attack state.evalFile(state.rootPath(CanonPath(flakeFile)), vInfo, false); // FIXME: symlink attack
expectType(state, nAttrs, vInfo, state.positions.add({CanonPath(flakeFile)}, 1, 1)); expectType(state, nAttrs, vInfo, state.positions.add({state.rootPath(CanonPath(flakeFile))}, 1, 1));
if (auto description = vInfo.attrs->get(state.sDescription)) { if (auto description = vInfo.attrs->get(state.sDescription)) {
expectType(state, nString, *description->value, description->pos); expectType(state, nString, *description->value, description->pos);
@ -346,7 +346,7 @@ LockedFlake lockFlake(
// FIXME: symlink attack // FIXME: symlink attack
auto oldLockFile = LockFile::read( auto oldLockFile = LockFile::read(
lockFlags.referenceLockFilePath.value_or( lockFlags.referenceLockFilePath.value_or(
flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock")); state.store->toRealPath(flake.storePath) + "/" + flake.lockedRef.subdir + "/flake.lock"));
debug("old lock file: %s", oldLockFile); debug("old lock file: %s", oldLockFile);
@ -574,7 +574,7 @@ LockedFlake lockFlake(
oldLock oldLock
? std::dynamic_pointer_cast<const Node>(oldLock) ? std::dynamic_pointer_cast<const Node>(oldLock)
: LockFile::read( : LockFile::read(
inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root.get_ptr(), state.store->toRealPath(inputFlake.storePath) + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root.get_ptr(),
oldLock ? lockRootPath : inputPath, oldLock ? lockRootPath : inputPath,
localPath, localPath,
false); false);
@ -598,7 +598,7 @@ LockedFlake lockFlake(
}; };
// Bring in the current ref for relative path resolution if we have it // Bring in the current ref for relative path resolution if we have it
auto parentPath = canonPath(flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir, true); auto parentPath = canonPath(state.store->toRealPath(flake.storePath) + "/" + flake.lockedRef.subdir, true);
computeLocks( computeLocks(
flake.inputs, flake.inputs,
@ -729,7 +729,7 @@ void callFlake(EvalState & state,
emitTreeAttrs( emitTreeAttrs(
state, state,
*lockedFlake.flake.sourceInfo, lockedFlake.flake.storePath,
lockedFlake.flake.lockedRef.input, lockedFlake.flake.lockedRef.input,
*vRootSrc, *vRootSrc,
false, false,
@ -737,14 +737,10 @@ void callFlake(EvalState & state,
vRootSubdir->mkString(lockedFlake.flake.lockedRef.subdir); vRootSubdir->mkString(lockedFlake.flake.lockedRef.subdir);
if (!state.vCallFlake) { auto vCallFlake = state.allocValue();
state.vCallFlake = allocRootValue(state.allocValue()); state.evalFile(state.callFlakeInternal, *vCallFlake);
state.eval(state.parseExprFromString(
#include "call-flake.nix.gen.hh"
, CanonPath::root), **state.vCallFlake);
}
state.callFunction(**state.vCallFlake, *vLocks, *vTmp1, noPos); state.callFunction(*vCallFlake, *vLocks, *vTmp1, noPos);
state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos); state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos);
state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos); state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos);
} }
@ -893,7 +889,7 @@ Fingerprint LockedFlake::getFingerprint() const
// flake.sourceInfo.storePath for the fingerprint. // flake.sourceInfo.storePath for the fingerprint.
return hashString(htSHA256, return hashString(htSHA256,
fmt("%s;%s;%d;%d;%s", fmt("%s;%s;%d;%d;%s",
flake.sourceInfo->storePath.to_string(), flake.storePath.to_string(),
flake.lockedRef.subdir, flake.lockedRef.subdir,
flake.lockedRef.input.getRevCount().value_or(0), flake.lockedRef.input.getRevCount().value_or(0),
flake.lockedRef.input.getLastModified().value_or(0), flake.lockedRef.input.getLastModified().value_or(0),

View file

@ -10,8 +10,6 @@ namespace nix {
class EvalState; class EvalState;
namespace fetchers { struct Tree; }
namespace flake { namespace flake {
struct FlakeInput; struct FlakeInput;
@ -84,7 +82,7 @@ struct Flake
*/ */
bool forceDirty = false; bool forceDirty = false;
std::optional<std::string> description; std::optional<std::string> description;
std::shared_ptr<const fetchers::Tree> sourceInfo; StorePath storePath;
FlakeInputs inputs; FlakeInputs inputs;
/** /**
* 'nixConfig' attribute * 'nixConfig' attribute
@ -193,7 +191,7 @@ void callFlake(
void emitTreeAttrs( void emitTreeAttrs(
EvalState & state, EvalState & state,
const fetchers::Tree & tree, const StorePath & storePath,
const fetchers::Input & input, const fetchers::Input & input,
Value & v, Value & v,
bool emptyRevFallback = false, bool emptyRevFallback = false,

View file

@ -272,10 +272,10 @@ FlakeRef FlakeRef::fromAttrs(const fetchers::Attrs & attrs)
fetchers::maybeGetStrAttr(attrs, "dir").value_or("")); fetchers::maybeGetStrAttr(attrs, "dir").value_or(""));
} }
std::pair<fetchers::Tree, FlakeRef> FlakeRef::fetchTree(ref<Store> store) const std::pair<StorePath, FlakeRef> FlakeRef::fetchTree(ref<Store> store) const
{ {
auto [tree, lockedInput] = input.fetch(store); auto [storePath, lockedInput] = input.fetch(store);
return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)}; return {std::move(storePath), FlakeRef(std::move(lockedInput), subdir)};
} }
std::tuple<FlakeRef, std::string, ExtendedOutputsSpec> parseFlakeRefWithFragmentAndExtendedOutputsSpec( std::tuple<FlakeRef, std::string, ExtendedOutputsSpec> parseFlakeRefWithFragmentAndExtendedOutputsSpec(

View file

@ -63,7 +63,7 @@ struct FlakeRef
static FlakeRef fromAttrs(const fetchers::Attrs & attrs); static FlakeRef fromAttrs(const fetchers::Attrs & attrs);
std::pair<fetchers::Tree, FlakeRef> fetchTree(ref<Store> store) const; std::pair<StorePath, FlakeRef> fetchTree(ref<Store> store) const;
}; };
std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef); std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef);

View file

@ -2,8 +2,10 @@
#include "store-api.hh" #include "store-api.hh"
#include "url-parts.hh" #include "url-parts.hh"
#include <algorithm>
#include <iomanip> #include <iomanip>
#include <iterator>
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
namespace nix::flake { namespace nix::flake {
@ -45,16 +47,26 @@ StorePath LockedNode::computeStorePath(Store & store) const
return lockedRef.input.computeStorePath(store); return lockedRef.input.computeStorePath(store);
} }
std::shared_ptr<Node> LockFile::findInput(const InputPath & path)
{ static std::shared_ptr<Node> doFind(const ref<Node>& root, const InputPath & path, std::vector<InputPath>& visited) {
auto pos = root; auto pos = root;
auto found = std::find(visited.cbegin(), visited.cend(), path);
if(found != visited.end()) {
std::vector<std::string> cycle;
std::transform(found, visited.cend(), std::back_inserter(cycle), printInputPath);
cycle.push_back(printInputPath(path));
throw Error("follow cycle detected: [%s]", concatStringsSep(" -> ", cycle));
}
visited.push_back(path);
for (auto & elem : path) { for (auto & elem : path) {
if (auto i = get(pos->inputs, elem)) { if (auto i = get(pos->inputs, elem)) {
if (auto node = std::get_if<0>(&*i)) if (auto node = std::get_if<0>(&*i))
pos = *node; pos = *node;
else if (auto follows = std::get_if<1>(&*i)) { else if (auto follows = std::get_if<1>(&*i)) {
if (auto p = findInput(*follows)) if (auto p = doFind(root, *follows, visited))
pos = ref(p); pos = ref(p);
else else
return {}; return {};
@ -66,6 +78,12 @@ std::shared_ptr<Node> LockFile::findInput(const InputPath & path)
return pos; return pos;
} }
std::shared_ptr<Node> LockFile::findInput(const InputPath & path)
{
std::vector<InputPath> visited;
return doFind(root, path, visited);
}
LockFile::LockFile(const nlohmann::json & json, const Path & path) LockFile::LockFile(const nlohmann::json & json, const Path & path)
{ {
auto version = json.value("version", 0); auto version = json.value("version", 0);

View file

@ -20,7 +20,6 @@ MakeError(Abort, EvalError);
MakeError(TypeError, EvalError); MakeError(TypeError, EvalError);
MakeError(UndefinedVarError, Error); MakeError(UndefinedVarError, Error);
MakeError(MissingArgumentError, EvalError); MakeError(MissingArgumentError, EvalError);
MakeError(RestrictedPathError, Error);
/** /**
* Position objects. * Position objects.
@ -200,9 +199,13 @@ struct ExprString : Expr
struct ExprPath : Expr struct ExprPath : Expr
{ {
ref<InputAccessor> accessor;
std::string s; std::string s;
Value v; Value v;
ExprPath(std::string s) : s(std::move(s)) { v.mkPath(this->s.c_str()); }; ExprPath(ref<InputAccessor> accessor, std::string s) : accessor(accessor), s(std::move(s))
{
v.mkPath(&*accessor, this->s.c_str());
}
Value * maybeThunk(EvalState & state, Env & env) override; Value * maybeThunk(EvalState & state, Env & env) override;
COMMON_METHODS COMMON_METHODS
}; };

View file

@ -522,7 +522,7 @@ path_start
/* add back in the trailing '/' to the first segment */ /* add back in the trailing '/' to the first segment */
if ($1.p[$1.l-1] == '/' && $1.l > 1) if ($1.p[$1.l-1] == '/' && $1.l > 1)
path += "/"; path += "/";
$$ = new ExprPath(std::move(path)); $$ = new ExprPath(ref<InputAccessor>(data->state.rootFS), std::move(path));
} }
| HPATH { | HPATH {
if (evalSettings.pureEval) { if (evalSettings.pureEval) {
@ -532,7 +532,7 @@ path_start
); );
} }
Path path(getHome() + std::string($1.p + 1, $1.l - 1)); Path path(getHome() + std::string($1.p + 1, $1.l - 1));
$$ = new ExprPath(std::move(path)); $$ = new ExprPath(ref<InputAccessor>(data->state.rootFS), std::move(path));
} }
; ;
@ -648,9 +648,11 @@ formal
#include "eval.hh" #include "eval.hh"
#include "filetransfer.hh" #include "filetransfer.hh"
#include "fetchers.hh" #include "tarball.hh"
#include "store-api.hh" #include "store-api.hh"
#include "flake/flake.hh" #include "flake/flake.hh"
#include "fs-input-accessor.hh"
#include "memory-input-accessor.hh"
namespace nix { namespace nix {
@ -758,11 +760,11 @@ SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_
auto r = *rOpt; auto r = *rOpt;
Path res = suffix == "" ? r : concatStrings(r, "/", suffix); Path res = suffix == "" ? r : concatStrings(r, "/", suffix);
if (pathExists(res)) return CanonPath(canonPath(res)); if (pathExists(res)) return rootPath(CanonPath(canonPath(res)));
} }
if (hasPrefix(path, "nix/")) if (hasPrefix(path, "nix/"))
return CanonPath(concatStrings(corepkgsPrefix, path.substr(4))); return {corepkgsFS, CanonPath(path.substr(3))};
debugThrow(ThrownError({ debugThrow(ThrownError({
.msg = hintfmt(evalSettings.pureEval .msg = hintfmt(evalSettings.pureEval
@ -785,7 +787,7 @@ std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Pa
if (EvalSettings::isPseudoUrl(value)) { if (EvalSettings::isPseudoUrl(value)) {
try { try {
auto storePath = fetchers::downloadTarball( auto storePath = fetchers::downloadTarball(
store, EvalSettings::resolvePseudoUrl(value), "source", false).tree.storePath; store, EvalSettings::resolvePseudoUrl(value), "source", false).storePath;
res = { store->toRealPath(storePath) }; res = { store->toRealPath(storePath) };
} catch (FileTransferError & e) { } catch (FileTransferError & e) {
logWarning({ logWarning({
@ -799,7 +801,7 @@ std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Pa
experimentalFeatureSettings.require(Xp::Flakes); experimentalFeatureSettings.require(Xp::Flakes);
auto flakeRef = parseFlakeRef(value.substr(6), {}, true, false); auto flakeRef = parseFlakeRef(value.substr(6), {}, true, false);
debug("fetching flake search path element '%s''", value); debug("fetching flake search path element '%s''", value);
auto storePath = flakeRef.resolve(store).fetchTree(store).first.storePath; auto storePath = flakeRef.resolve(store).fetchTree(store).first;
res = { store->toRealPath(storePath) }; res = { store->toRealPath(storePath) };
} }

View file

@ -1,10 +1,11 @@
#include "eval.hh" #include "eval.hh"
#include "fs-input-accessor.hh"
namespace nix { namespace nix {
SourcePath EvalState::rootPath(CanonPath path) SourcePath EvalState::rootPath(CanonPath path)
{ {
return std::move(path); return {rootFS, std::move(path)};
} }
} }

View file

@ -121,13 +121,15 @@ static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, co
auto path = state.coerceToPath(noPos, v, context, "while realising the context of a path"); auto path = state.coerceToPath(noPos, v, context, "while realising the context of a path");
try { try {
StringMap rewrites = state.realiseContext(context); if (!context.empty()) {
auto rewrites = state.realiseContext(context);
auto realPath = state.rootPath(CanonPath(state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context))); auto realPath = state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context);
return {path.accessor, CanonPath(realPath)};
}
return flags.checkForPureEval return flags.checkForPureEval
? state.checkSourcePath(realPath) ? state.checkSourcePath(path)
: realPath; : path;
} catch (Error & e) { } catch (Error & e) {
e.addTrace(state.positions[pos], "while realising the context of path '%s'", path); e.addTrace(state.positions[pos], "while realising the context of path '%s'", path);
throw; throw;
@ -202,7 +204,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v
state.vImportedDrvToDerivation = allocRootValue(state.allocValue()); state.vImportedDrvToDerivation = allocRootValue(state.allocValue());
state.eval(state.parseExprFromString( state.eval(state.parseExprFromString(
#include "imported-drv-to-derivation.nix.gen.hh" #include "imported-drv-to-derivation.nix.gen.hh"
, CanonPath::root), **state.vImportedDrvToDerivation); , state.rootPath(CanonPath::root)), **state.vImportedDrvToDerivation);
} }
state.forceFunction(**state.vImportedDrvToDerivation, pos, "while evaluating imported-drv-to-derivation.nix.gen.hh"); state.forceFunction(**state.vImportedDrvToDerivation, pos, "while evaluating imported-drv-to-derivation.nix.gen.hh");
@ -210,12 +212,6 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v
state.forceAttrs(v, pos, "while calling imported-drv-to-derivation.nix.gen.hh"); state.forceAttrs(v, pos, "while calling imported-drv-to-derivation.nix.gen.hh");
} }
else if (path2 == corepkgsPrefix + "fetchurl.nix") {
state.eval(state.parseExprFromString(
#include "fetchurl.nix.gen.hh"
, CanonPath::root), v);
}
else { else {
if (!vScope) if (!vScope)
state.evalFile(path, v); state.evalFile(path, v);
@ -599,7 +595,10 @@ struct CompareValues
case nString: case nString:
return v1->string_view().compare(v2->string_view()) < 0; return v1->string_view().compare(v2->string_view()) < 0;
case nPath: case nPath:
return strcmp(v1->_path, v2->_path) < 0; // Note: we don't take the accessor into account
// since it's not obvious how to compare them in a
// reproducible way.
return strcmp(v1->_path.path, v2->_path.path) < 0;
case nList: case nList:
// Lexicographic comparison // Lexicographic comparison
for (size_t i = 0;; i++) { for (size_t i = 0;; i++) {
@ -734,6 +733,14 @@ static RegisterPrimOp primop_genericClosure(PrimOp {
``` ```
[ { key = 5; } { key = 16; } { key = 8; } { key = 4; } { key = 2; } { key = 1; } ] [ { key = 5; } { key = 16; } { key = 8; } { key = 4; } { key = 2; } { key = 1; } ]
``` ```
`key` can be one of the following types:
- [Number](@docroot@/language/values.md#type-number)
- [Boolean](@docroot@/language/values.md#type-boolean)
- [String](@docroot@/language/values.md#type-string)
- [Path](@docroot@/language/values.md#type-path)
- [List](@docroot@/language/values.md#list)
)", )",
.fun = prim_genericClosure, .fun = prim_genericClosure,
}); });
@ -1485,7 +1492,7 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args,
})); }));
NixStringContext context; NixStringContext context;
auto path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.storePath")).path; auto path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'")).path;
/* Resolve symlinks in path, unless path itself is a symlink /* Resolve symlinks in path, unless path itself is a symlink
directly in the store. The latter condition is necessary so directly in the store. The latter condition is necessary so
e.g. nix-push does the right thing. */ e.g. nix-push does the right thing. */
@ -1758,7 +1765,7 @@ static void prim_hashFile(EvalState & state, const PosIdx pos, Value * * args, V
auto path = realisePath(state, pos, *args[1]); auto path = realisePath(state, pos, *args[1]);
v.mkString(hashString(*ht, path.readFile()).to_string(Base16, false)); v.mkString(hashString(*ht, path.readFile()).to_string(HashFormat::Base16, false));
} }
static RegisterPrimOp primop_hashFile({ static RegisterPrimOp primop_hashFile({
@ -2203,7 +2210,7 @@ static void addPath(
path = evalSettings.pureEval && expectedHash path = evalSettings.pureEval && expectedHash
? path ? path
: state.checkSourcePath(CanonPath(path)).path.abs(); : state.checkSourcePath(state.rootPath(CanonPath(path))).path.abs();
PathFilter filter = filterFun ? ([&](const Path & path) { PathFilter filter = filterFun ? ([&](const Path & path) {
auto st = lstat(path); auto st = lstat(path);
@ -2236,9 +2243,7 @@ static void addPath(
}); });
if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
StorePath dstPath = settings.readOnlyMode auto dstPath = state.rootPath(CanonPath(path)).fetchToStore(state.store, name, method, &filter, state.repair);
? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first
: state.store->addToStore(name, path, method, htSHA256, filter, state.repair, refs);
if (expectedHash && expectedStorePath != dstPath) if (expectedHash && expectedStorePath != dstPath)
state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path)); state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path));
state.allowAndSetStorePathString(dstPath, v); state.allowAndSetStorePathString(dstPath, v);
@ -2255,7 +2260,7 @@ static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * arg
{ {
NixStringContext context; NixStringContext context;
auto path = state.coerceToPath(pos, *args[1], context, auto path = state.coerceToPath(pos, *args[1], context,
"while evaluating the second argument (the path to filter) passed to builtins.filterSource"); "while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'");
state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource"); state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource");
addPath(state, pos, path.baseName(), path.path.abs(), args[0], FileIngestionMethod::Recursive, std::nullopt, v, context); addPath(state, pos, path.baseName(), path.path.abs(), args[0], FileIngestionMethod::Recursive, std::nullopt, v, context);
} }
@ -3715,10 +3720,11 @@ static RegisterPrimOp primop_substring({
.doc = R"( .doc = R"(
Return the substring of *s* from character position *start* Return the substring of *s* from character position *start*
(zero-based) up to but not including *start + len*. If *start* is (zero-based) up to but not including *start + len*. If *start* is
greater than the length of the string, an empty string is returned, greater than the length of the string, an empty string is returned.
and if *start + len* lies beyond the end of the string, only the If *start + len* lies beyond the end of the string or *len* is `-1`,
substring up to the end of the string is returned. *start* must be only the substring up to the end of the string is returned.
non-negative. For example, *start* must be non-negative.
For example,
```nix ```nix
builtins.substring 0 3 "nixos" builtins.substring 0 3 "nixos"
@ -3760,7 +3766,7 @@ static void prim_hashString(EvalState & state, const PosIdx pos, Value * * args,
NixStringContext context; // discarded NixStringContext context; // discarded
auto s = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString"); auto s = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString");
v.mkString(hashString(*ht, s).to_string(Base16, false)); v.mkString(hashString(*ht, s).to_string(HashFormat::Base16, false));
} }
static RegisterPrimOp primop_hashString({ static RegisterPrimOp primop_hashString({
@ -3774,6 +3780,101 @@ static RegisterPrimOp primop_hashString({
.fun = prim_hashString, .fun = prim_hashString,
}); });
static void prim_convertHash(EvalState & state, const PosIdx pos, Value * * args, Value & v)
{
state.forceAttrs(*args[0], pos, "while evaluating the first argument passed to builtins.convertHash");
auto &inputAttrs = args[0]->attrs;
Bindings::iterator iteratorHash = getAttr(state, state.symbols.create("hash"), inputAttrs, "while locating the attribute 'hash'");
auto hash = state.forceStringNoCtx(*iteratorHash->value, pos, "while evaluating the attribute 'hash'");
Bindings::iterator iteratorHashAlgo = inputAttrs->find(state.symbols.create("hashAlgo"));
std::optional<HashType> ht = std::nullopt;
if (iteratorHashAlgo != inputAttrs->end()) {
ht = parseHashType(state.forceStringNoCtx(*iteratorHashAlgo->value, pos, "while evaluating the attribute 'hashAlgo'"));
}
Bindings::iterator iteratorToHashFormat = getAttr(state, state.symbols.create("toHashFormat"), args[0]->attrs, "while locating the attribute 'toHashFormat'");
HashFormat hf = parseHashFormat(state.forceStringNoCtx(*iteratorToHashFormat->value, pos, "while evaluating the attribute 'toHashFormat'"));
v.mkString(Hash::parseAny(hash, ht).to_string(hf, hf == HashFormat::SRI));
}
static RegisterPrimOp primop_convertHash({
.name = "__convertHash",
.args = {"args"},
.doc = R"(
Return the specified representation of a hash string, based on the attributes presented in *args*:
- `hash`
The hash to be converted.
The hash format is detected automatically.
- `hashAlgo`
The algorithm used to create the hash. Must be one of
- `"md5"`
- `"sha1"`
- `"sha256"`
- `"sha512"`
The attribute may be omitted when `hash` is an [SRI hash](https://www.w3.org/TR/SRI/#the-integrity-attribute) or when the hash is prefixed with the hash algorithm name followed by a colon.
That `<hashAlgo>:<hashBody>` syntax is supported for backwards compatibility with existing tooling.
- `toHashFormat`
The format of the resulting hash. Must be one of
- `"base16"`
- `"base32"`
- `"base64"`
- `"sri"`
The result hash is the *toHashFormat* representation of the hash *hash*.
> **Example**
>
> Convert a SHA256 hash in Base16 to SRI:
>
> ```nix
> builtins.convertHash {
> hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
> toHashFormat = "sri";
> hashAlgo = "sha256";
> }
> ```
>
> "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="
> **Example**
>
> Convert a SHA256 hash in SRI to Base16:
>
> ```nix
> builtins.convertHash {
> hash = "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=";
> toHashFormat = "base16";
> }
> ```
>
> "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
> **Example**
>
> Convert a hash in the form `<hashAlgo>:<hashBody>` in Base16 to SRI:
>
> ```nix
> builtins.convertHash {
> hash = "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
> toHashFormat = "sri";
> }
> ```
>
> "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="
)",
.fun = prim_convertHash,
});
struct RegexCache struct RegexCache
{ {
// TODO use C++20 transparent comparison when available // TODO use C++20 transparent comparison when available
@ -4450,12 +4551,7 @@ void EvalState::createBaseEnv()
/* Note: we have to initialize the 'derivation' constant *after* /* Note: we have to initialize the 'derivation' constant *after*
building baseEnv/staticBaseEnv because it uses 'builtins'. */ building baseEnv/staticBaseEnv because it uses 'builtins'. */
char code[] = evalFile(derivationInternal, *vDerivation);
#include "primops/derivation.nix.gen.hh"
// the parser needs two NUL bytes as terminators; one of them
// is implied by being a C string.
"\0";
eval(parse(code, sizeof(code), derivationInternal, {CanonPath::root}, staticBaseEnv), *vDerivation);
} }

View file

@ -30,20 +30,27 @@ static RegisterPrimOp primop_hasContext({
.name = "__hasContext", .name = "__hasContext",
.args = {"s"}, .args = {"s"},
.doc = R"( .doc = R"(
Return `true` if string *s* has a non-empty context. The Return `true` if string *s* has a non-empty context.
context can be obtained with The context can be obtained with
[`getContext`](#builtins-getContext). [`getContext`](#builtins-getContext).
> **Example**
>
> Many operations require a string context to be empty because they are intended only to work with "regular" strings, and also to help users avoid unintentionally loosing track of string context elements.
> `builtins.hasContext` can help create better domain-specific errors in those case.
>
> ```nix
> name: meta:
>
> if builtins.hasContext name
> then throw "package name cannot contain string context"
> else { ${name} = meta; }
> ```
)", )",
.fun = prim_hasContext .fun = prim_hasContext
}); });
/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a
builder without causing the derivation to be built (for instance,
in the derivation that builds NARs in nix-push, when doing
source-only deployment). This primop marks the string context so
that builtins.derivation adds the path to drv.inputSrcs rather than
drv.inputDrvs. */
static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx pos, Value * * args, Value & v) static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx pos, Value * * args, Value & v)
{ {
NixStringContext context; NixStringContext context;
@ -66,11 +73,83 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx p
static RegisterPrimOp primop_unsafeDiscardOutputDependency({ static RegisterPrimOp primop_unsafeDiscardOutputDependency({
.name = "__unsafeDiscardOutputDependency", .name = "__unsafeDiscardOutputDependency",
.arity = 1, .args = {"s"},
.doc = R"(
Create a copy of the given string where every "derivation deep" string context element is turned into a constant string context element.
This is the opposite of [`builtins.addDrvOutputDependencies`](#builtins-addDrvOutputDependencies).
This is unsafe because it allows us to "forget" store objects we would have otherwise refered to with the string context,
whereas Nix normally tracks all dependencies consistently.
Safe operations "grow" but never "shrink" string contexts.
[`builtins.addDrvOutputDependencies`] in contrast is safe because "derivation deep" string context element always refers to the underlying derivation (among many more things).
Replacing a constant string context element with a "derivation deep" element is a safe operation that just enlargens the string context without forgetting anything.
[`builtins.addDrvOutputDependencies`]: #builtins-addDrvOutputDependencies
)",
.fun = prim_unsafeDiscardOutputDependency .fun = prim_unsafeDiscardOutputDependency
}); });
static void prim_addDrvOutputDependencies(EvalState & state, const PosIdx pos, Value * * args, Value & v)
{
NixStringContext context;
auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.addDrvOutputDependencies");
auto contextSize = context.size();
if (contextSize != 1) {
throw EvalError({
.msg = hintfmt("context of string '%s' must have exactly one element, but has %d", *s, contextSize),
.errPos = state.positions[pos]
});
}
NixStringContext context2 {
(NixStringContextElem { std::visit(overloaded {
[&](const NixStringContextElem::Opaque & c) -> NixStringContextElem::DrvDeep {
if (!c.path.isDerivation()) {
throw EvalError({
.msg = hintfmt("path '%s' is not a derivation",
state.store->printStorePath(c.path)),
.errPos = state.positions[pos],
});
}
return NixStringContextElem::DrvDeep {
.drvPath = c.path,
};
},
[&](const NixStringContextElem::Built & c) -> NixStringContextElem::DrvDeep {
throw EvalError({
.msg = hintfmt("`addDrvOutputDependencies` can only act on derivations, not on a derivation output such as '%1%'", c.output),
.errPos = state.positions[pos],
});
},
[&](const NixStringContextElem::DrvDeep & c) -> NixStringContextElem::DrvDeep {
/* Reuse original item because we want this to be idempotent. */
return std::move(c);
},
}, context.begin()->raw) }),
};
v.mkString(*s, context2);
}
static RegisterPrimOp primop_addDrvOutputDependencies({
.name = "__addDrvOutputDependencies",
.args = {"s"},
.doc = R"(
Create a copy of the given string where a single consant string context element is turned into a "derivation deep" string context element.
The store path that is the constant string context element should point to a valid derivation, and end in `.drv`.
The original string context element must not be empty or have multiple elements, and it must not have any other type of element other than a constant or derivation deep element.
The latter is supported so this function is idempotent.
This is the opposite of [`builtins.unsafeDiscardOutputDependency`](#builtins-addDrvOutputDependencies).
)",
.fun = prim_addDrvOutputDependencies
});
/* Extract the context of a string as a structured Nix value. /* Extract the context of a string as a structured Nix value.
The context is represented as an attribute set whose keys are the The context is represented as an attribute set whose keys are the

View file

@ -71,10 +71,10 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a
auto input = fetchers::Input::fromAttrs(std::move(attrs)); auto input = fetchers::Input::fromAttrs(std::move(attrs));
// FIXME: use name // FIXME: use name
auto [tree, input2] = input.fetch(state.store); auto [storePath, input2] = input.fetch(state.store);
auto attrs2 = state.buildBindings(8); auto attrs2 = state.buildBindings(8);
state.mkStorePathString(tree.storePath, attrs2.alloc(state.sOutPath)); state.mkStorePathString(storePath, attrs2.alloc(state.sOutPath));
if (input2.getRef()) if (input2.getRef())
attrs2.alloc("branch").mkString(*input2.getRef()); attrs2.alloc("branch").mkString(*input2.getRef());
// Backward compatibility: set 'rev' to // Backward compatibility: set 'rev' to
@ -86,7 +86,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a
attrs2.alloc("revCount").mkInt(*revCount); attrs2.alloc("revCount").mkInt(*revCount);
v.mkAttrs(attrs2); v.mkAttrs(attrs2);
state.allowPath(tree.storePath); state.allowPath(storePath);
} }
static RegisterPrimOp r_fetchMercurial({ static RegisterPrimOp r_fetchMercurial({

View file

@ -5,6 +5,7 @@
#include "fetchers.hh" #include "fetchers.hh"
#include "filetransfer.hh" #include "filetransfer.hh"
#include "registry.hh" #include "registry.hh"
#include "tarball.hh"
#include "url.hh" #include "url.hh"
#include <ctime> #include <ctime>
@ -15,7 +16,7 @@ namespace nix {
void emitTreeAttrs( void emitTreeAttrs(
EvalState & state, EvalState & state,
const fetchers::Tree & tree, const StorePath & storePath,
const fetchers::Input & input, const fetchers::Input & input,
Value & v, Value & v,
bool emptyRevFallback, bool emptyRevFallback,
@ -25,13 +26,13 @@ void emitTreeAttrs(
auto attrs = state.buildBindings(10); auto attrs = state.buildBindings(10);
state.mkStorePathString(tree.storePath, attrs.alloc(state.sOutPath)); state.mkStorePathString(storePath, attrs.alloc(state.sOutPath));
// FIXME: support arbitrary input attributes. // FIXME: support arbitrary input attributes.
auto narHash = input.getNarHash(); auto narHash = input.getNarHash();
assert(narHash); assert(narHash);
attrs.alloc("narHash").mkString(narHash->to_string(SRI, true)); attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true));
if (input.getType() == "git") if (input.getType() == "git")
attrs.alloc("submodules").mkBool( attrs.alloc("submodules").mkBool(
@ -148,6 +149,11 @@ static void fetchTree(
attrs.emplace("url", fixGitURL(url)); attrs.emplace("url", fixGitURL(url));
input = fetchers::Input::fromAttrs(std::move(attrs)); input = fetchers::Input::fromAttrs(std::move(attrs));
} else { } else {
if (!experimentalFeatureSettings.isEnabled(Xp::Flakes))
state.debugThrowLastTrace(EvalError({
.msg = hintfmt("passing a string argument to 'fetchTree' requires the 'flakes' experimental feature"),
.errPos = state.positions[pos]
}));
input = fetchers::Input::fromURL(url); input = fetchers::Input::fromURL(url);
} }
} }
@ -160,11 +166,11 @@ static void fetchTree(
state.checkURI(input.toURLString()); state.checkURI(input.toURLString());
auto [tree, input2] = input.fetch(state.store); auto [storePath, input2] = input.fetch(state.store);
state.allowPath(tree.storePath); state.allowPath(storePath);
emitTreeAttrs(state, tree, input2, v, params.emptyRevFallback, false); emitTreeAttrs(state, storePath, input2, v, params.emptyRevFallback, false);
} }
static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args, Value & v) static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args, Value & v)
@ -180,6 +186,10 @@ static RegisterPrimOp primop_fetchTree({
*input* must be a [flake reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references), either in attribute set representation or in the URL-like syntax. *input* must be a [flake reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references), either in attribute set representation or in the URL-like syntax.
The input should be "locked", that is, it should contain a commit hash or content hash unless impure evaluation (`--impure`) is enabled. The input should be "locked", that is, it should contain a commit hash or content hash unless impure evaluation (`--impure`) is enabled.
> **Note**
>
> The URL-like syntax requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled.
Here are some examples of how to use `fetchTree`: Here are some examples of how to use `fetchTree`:
- Fetch a GitHub repository using the attribute set representation: - Fetch a GitHub repository using the attribute set representation:
@ -213,7 +223,6 @@ static RegisterPrimOp primop_fetchTree({
``` ```
)", )",
.fun = prim_fetchTree, .fun = prim_fetchTree,
.experimentalFeature = Xp::Flakes,
}); });
static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v, static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v,
@ -280,7 +289,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
// https://github.com/NixOS/nix/issues/4313 // https://github.com/NixOS/nix/issues/4313
auto storePath = auto storePath =
unpack unpack
? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).tree.storePath ? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath
: fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath; : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath;
if (expectedHash) { if (expectedHash) {
@ -289,7 +298,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
: hashFile(htSHA256, state.store->toRealPath(storePath)); : hashFile(htSHA256, state.store->toRealPath(storePath));
if (hash != *expectedHash) if (hash != *expectedHash)
state.debugThrowLastTrace(EvalError((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s", state.debugThrowLastTrace(EvalError((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s",
*url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true))); *url, expectedHash->to_string(HashFormat::Base32, true), hash.to_string(HashFormat::Base32, true)));
} }
state.allowAndSetStorePathString(storePath, v); state.allowAndSetStorePathString(storePath, v);
@ -383,7 +392,7 @@ static RegisterPrimOp primop_fetchGit({
The URL of the repo. The URL of the repo.
- `name` (default: *basename of the URL*) - `name` (default: `source`)
The name of the directory the repo should be exported to in the store. The name of the directory the repo should be exported to in the store.

View file

@ -310,7 +310,7 @@ namespace nix {
ASSERT_TRACE2("storePath true", ASSERT_TRACE2("storePath true",
TypeError, TypeError,
hintfmt("cannot coerce %s to a string", "a Boolean"), hintfmt("cannot coerce %s to a string", "a Boolean"),
hintfmt("while evaluating the first argument passed to builtins.storePath")); hintfmt("while evaluating the first argument passed to 'builtins.storePath'"));
} }
@ -378,12 +378,12 @@ namespace nix {
ASSERT_TRACE2("filterSource [] []", ASSERT_TRACE2("filterSource [] []",
TypeError, TypeError,
hintfmt("cannot coerce %s to a string", "a list"), hintfmt("cannot coerce %s to a string", "a list"),
hintfmt("while evaluating the second argument (the path to filter) passed to builtins.filterSource")); hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'"));
ASSERT_TRACE2("filterSource [] \"foo\"", ASSERT_TRACE2("filterSource [] \"foo\"",
EvalError, EvalError,
hintfmt("string '%s' doesn't represent an absolute path", "foo"), hintfmt("string '%s' doesn't represent an absolute path", "foo"),
hintfmt("while evaluating the second argument (the path to filter) passed to builtins.filterSource")); hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'"));
ASSERT_TRACE2("filterSource [] ./.", ASSERT_TRACE2("filterSource [] ./.",
TypeError, TypeError,
@ -1084,7 +1084,7 @@ namespace nix {
ASSERT_TRACE1("hashString \"foo\" \"content\"", ASSERT_TRACE1("hashString \"foo\" \"content\"",
UsageError, UsageError,
hintfmt("unknown hash algorithm '%s'", "foo")); hintfmt("unknown hash algorithm '%s', expect 'md5', 'sha1', 'sha256', or 'sha512'", "foo"));
ASSERT_TRACE2("hashString \"sha256\" {}", ASSERT_TRACE2("hashString \"sha256\" {}",
TypeError, TypeError,

View file

@ -62,7 +62,7 @@ namespace nix {
// not supported by store 'dummy'" thrown in the test body. // not supported by store 'dummy'" thrown in the test body.
TEST_F(JSONValueTest, DISABLED_Path) { TEST_F(JSONValueTest, DISABLED_Path) {
Value v; Value v;
v.mkPath("test"); v.mkPath(state.rootPath(CanonPath("/test")));
ASSERT_EQ(getJSONValue(v), "\"/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x\""); ASSERT_EQ(getJSONValue(v), "\"/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x\"");
} }
} /* namespace nix */ } /* namespace nix */

View file

@ -106,10 +106,13 @@ namespace nix {
if (arg.type() != nPath) { if (arg.type() != nPath) {
*result_listener << "Expected a path got " << arg.type(); *result_listener << "Expected a path got " << arg.type();
return false; return false;
} else if (std::string_view(arg._path) != p) { } else {
*result_listener << "Expected a path that equals \"" << p << "\" but got: " << arg.c_str(); auto path = arg.path();
if (path.path != CanonPath(p)) {
*result_listener << "Expected a path that equals \"" << p << "\" but got: " << path.path;
return false; return false;
} }
}
return true; return true;
} }

View file

@ -189,7 +189,12 @@ public:
const char * c_str; const char * c_str;
const char * * context; // must be in sorted order const char * * context; // must be in sorted order
} string; } string;
const char * _path;
struct {
InputAccessor * accessor;
const char * path;
} _path;
Bindings * attrs; Bindings * attrs;
struct { struct {
size_t size; size_t size;
@ -286,11 +291,12 @@ public:
void mkPath(const SourcePath & path); void mkPath(const SourcePath & path);
inline void mkPath(const char * path) inline void mkPath(InputAccessor * accessor, const char * path)
{ {
clearValue(); clearValue();
internalType = tPath; internalType = tPath;
_path = path; _path.accessor = accessor;
_path.path = path;
} }
inline void mkNull() inline void mkNull()
@ -437,7 +443,10 @@ public:
SourcePath path() const SourcePath path() const
{ {
assert(internalType == tPath); assert(internalType == tPath);
return SourcePath{CanonPath(_path)}; return SourcePath {
.accessor = ref(_path.accessor->shared_from_this()),
.path = CanonPath(CanonPath::unchecked_t(), _path.path)
};
} }
std::string_view string_view() const std::string_view string_view() const

View file

@ -84,16 +84,16 @@ std::string Input::to_string() const
return toURL().to_string(); return toURL().to_string();
} }
bool Input::isDirect() const
{
return !scheme || scheme->isDirect(*this);
}
Attrs Input::toAttrs() const Attrs Input::toAttrs() const
{ {
return attrs; return attrs;
} }
bool Input::hasAllInfo() const
{
return getNarHash() && scheme && scheme->hasAllInfo(*this);
}
bool Input::operator ==(const Input & other) const bool Input::operator ==(const Input & other) const
{ {
return attrs == other.attrs; return attrs == other.attrs;
@ -109,7 +109,7 @@ bool Input::contains(const Input & other) const
return false; return false;
} }
std::pair<Tree, Input> Input::fetch(ref<Store> store) const std::pair<StorePath, Input> Input::fetch(ref<Store> store) const
{ {
if (!scheme) if (!scheme)
throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs())); throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs()));
@ -117,7 +117,7 @@ std::pair<Tree, Input> Input::fetch(ref<Store> store) const
/* The tree may already be in the Nix store, or it could be /* The tree may already be in the Nix store, or it could be
substituted (which is often faster than fetching from the substituted (which is often faster than fetching from the
original source). So check that. */ original source). So check that. */
if (hasAllInfo()) { if (getNarHash()) {
try { try {
auto storePath = computeStorePath(*store); auto storePath = computeStorePath(*store);
@ -126,7 +126,7 @@ std::pair<Tree, Input> Input::fetch(ref<Store> store) const
debug("using substituted/cached input '%s' in '%s'", debug("using substituted/cached input '%s' in '%s'",
to_string(), store->printStorePath(storePath)); to_string(), store->printStorePath(storePath));
return {Tree { .actualPath = store->toRealPath(storePath), .storePath = std::move(storePath) }, *this}; return {std::move(storePath), *this};
} catch (Error & e) { } catch (Error & e) {
debug("substitution of input '%s' failed: %s", to_string(), e.what()); debug("substitution of input '%s' failed: %s", to_string(), e.what());
} }
@ -141,18 +141,16 @@ std::pair<Tree, Input> Input::fetch(ref<Store> store) const
} }
}(); }();
Tree tree { auto narHash = store->queryPathInfo(storePath)->narHash;
.actualPath = store->toRealPath(storePath), input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
.storePath = storePath,
};
auto narHash = store->queryPathInfo(tree.storePath)->narHash;
input.attrs.insert_or_assign("narHash", narHash.to_string(SRI, true));
if (auto prevNarHash = getNarHash()) { if (auto prevNarHash = getNarHash()) {
if (narHash != *prevNarHash) if (narHash != *prevNarHash)
throw Error((unsigned int) 102, "NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'", throw Error((unsigned int) 102, "NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash.to_string(SRI, true)); to_string(),
store->printStorePath(storePath),
prevNarHash->to_string(HashFormat::SRI, true),
narHash.to_string(HashFormat::SRI, true));
} }
if (auto prevLastModified = getLastModified()) { if (auto prevLastModified = getLastModified()) {
@ -175,9 +173,7 @@ std::pair<Tree, Input> Input::fetch(ref<Store> store) const
input.locked = true; input.locked = true;
assert(input.hasAllInfo()); return {std::move(storePath), input};
return {std::move(tree), input};
} }
Input Input::applyOverrides( Input Input::applyOverrides(

View file

@ -13,12 +13,6 @@ namespace nix { class Store; }
namespace nix::fetchers { namespace nix::fetchers {
struct Tree
{
Path actualPath;
StorePath storePath;
};
struct InputScheme; struct InputScheme;
/** /**
@ -35,7 +29,6 @@ struct Input
std::shared_ptr<InputScheme> scheme; // note: can be null std::shared_ptr<InputScheme> scheme; // note: can be null
Attrs attrs; Attrs attrs;
bool locked = false; bool locked = false;
bool direct = true;
/** /**
* path of the parent of this input, used for relative path resolution * path of the parent of this input, used for relative path resolution
@ -71,7 +64,7 @@ public:
* Check whether this is a "direct" input, that is, not * Check whether this is a "direct" input, that is, not
* one that goes through a registry. * one that goes through a registry.
*/ */
bool isDirect() const { return direct; } bool isDirect() const;
/** /**
* Check whether this is a "locked" input, that is, * Check whether this is a "locked" input, that is,
@ -79,24 +72,15 @@ public:
*/ */
bool isLocked() const { return locked; } bool isLocked() const { return locked; }
/**
* Check whether the input carries all necessary info required
* for cache insertion and substitution.
* These fields are used to uniquely identify cached trees
* within the "tarball TTL" window without necessarily
* indicating that the input's origin is unchanged.
*/
bool hasAllInfo() const;
bool operator ==(const Input & other) const; bool operator ==(const Input & other) const;
bool contains(const Input & other) const; bool contains(const Input & other) const;
/** /**
* Fetch the input into the Nix store, returning the location in * Fetch the entire input into the Nix store, returning the
* the Nix store and the locked input. * location in the Nix store and the locked input.
*/ */
std::pair<Tree, Input> fetch(ref<Store> store) const; std::pair<StorePath, Input> fetch(ref<Store> store) const;
Input applyOverrides( Input applyOverrides(
std::optional<std::string> ref, std::optional<std::string> ref,
@ -144,8 +128,6 @@ struct InputScheme
virtual ParsedURL toURL(const Input & input) const; virtual ParsedURL toURL(const Input & input) const;
virtual bool hasAllInfo(const Input & input) const = 0;
virtual Input applyOverrides( virtual Input applyOverrides(
const Input & input, const Input & input,
std::optional<std::string> ref, std::optional<std::string> ref,
@ -163,37 +145,11 @@ struct InputScheme
* Is this `InputScheme` part of an experimental feature? * Is this `InputScheme` part of an experimental feature?
*/ */
virtual std::optional<ExperimentalFeature> experimentalFeature(); virtual std::optional<ExperimentalFeature> experimentalFeature();
virtual bool isDirect(const Input & input) const
{ return true; }
}; };
void registerInputScheme(std::shared_ptr<InputScheme> && fetcher); void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);
struct DownloadFileResult
{
StorePath storePath;
std::string etag;
std::string effectiveUrl;
std::optional<std::string> immutableUrl;
};
DownloadFileResult downloadFile(
ref<Store> store,
const std::string & url,
const std::string & name,
bool locked,
const Headers & headers = {});
struct DownloadTarballResult
{
Tree tree;
time_t lastModified;
std::optional<std::string> immutableUrl;
};
DownloadTarballResult downloadTarball(
ref<Store> store,
const std::string & url,
const std::string & name,
bool locked,
const Headers & headers = {});
} }

View file

@ -0,0 +1,130 @@
#include "fs-input-accessor.hh"
#include "posix-source-accessor.hh"
#include "store-api.hh"
namespace nix {
struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor
{
CanonPath root;
std::optional<std::set<CanonPath>> allowedPaths;
MakeNotAllowedError makeNotAllowedError;
FSInputAccessorImpl(
const CanonPath & root,
std::optional<std::set<CanonPath>> && allowedPaths,
MakeNotAllowedError && makeNotAllowedError)
: root(root)
, allowedPaths(std::move(allowedPaths))
, makeNotAllowedError(std::move(makeNotAllowedError))
{
}
void readFile(
const CanonPath & path,
Sink & sink,
std::function<void(uint64_t)> sizeCallback) override
{
auto absPath = makeAbsPath(path);
checkAllowed(absPath);
PosixSourceAccessor::readFile(absPath, sink, sizeCallback);
}
bool pathExists(const CanonPath & path) override
{
auto absPath = makeAbsPath(path);
return isAllowed(absPath) && PosixSourceAccessor::pathExists(absPath);
}
Stat lstat(const CanonPath & path) override
{
auto absPath = makeAbsPath(path);
checkAllowed(absPath);
return PosixSourceAccessor::lstat(absPath);
}
DirEntries readDirectory(const CanonPath & path) override
{
auto absPath = makeAbsPath(path);
checkAllowed(absPath);
DirEntries res;
for (auto & entry : PosixSourceAccessor::readDirectory(absPath))
if (isAllowed(absPath + entry.first))
res.emplace(entry);
return res;
}
std::string readLink(const CanonPath & path) override
{
auto absPath = makeAbsPath(path);
checkAllowed(absPath);
return PosixSourceAccessor::readLink(absPath);
}
CanonPath makeAbsPath(const CanonPath & path)
{
return root + path;
}
void checkAllowed(const CanonPath & absPath) override
{
if (!isAllowed(absPath))
throw makeNotAllowedError
? makeNotAllowedError(absPath)
: RestrictedPathError("access to path '%s' is forbidden", absPath);
}
bool isAllowed(const CanonPath & absPath)
{
if (!absPath.isWithin(root))
return false;
if (allowedPaths) {
auto p = absPath.removePrefix(root);
if (!p.isAllowed(*allowedPaths))
return false;
}
return true;
}
void allowPath(CanonPath path) override
{
if (allowedPaths)
allowedPaths->insert(std::move(path));
}
bool hasAccessControl() override
{
return (bool) allowedPaths;
}
std::optional<CanonPath> getPhysicalPath(const CanonPath & path) override
{
return makeAbsPath(path);
}
};
ref<FSInputAccessor> makeFSInputAccessor(
const CanonPath & root,
std::optional<std::set<CanonPath>> && allowedPaths,
MakeNotAllowedError && makeNotAllowedError)
{
return make_ref<FSInputAccessorImpl>(root, std::move(allowedPaths), std::move(makeNotAllowedError));
}
ref<FSInputAccessor> makeStorePathAccessor(
ref<Store> store,
const StorePath & storePath,
MakeNotAllowedError && makeNotAllowedError)
{
return makeFSInputAccessor(CanonPath(store->toRealPath(storePath)), {}, std::move(makeNotAllowedError));
}
SourcePath getUnfilteredRootPath(CanonPath path)
{
static auto rootFS = makeFSInputAccessor(CanonPath::root);
return {rootFS, path};
}
}

View file

@ -0,0 +1,33 @@
#pragma once
#include "input-accessor.hh"
namespace nix {
class StorePath;
class Store;
struct FSInputAccessor : InputAccessor
{
virtual void checkAllowed(const CanonPath & absPath) = 0;
virtual void allowPath(CanonPath path) = 0;
virtual bool hasAccessControl() = 0;
};
typedef std::function<RestrictedPathError(const CanonPath & path)> MakeNotAllowedError;
ref<FSInputAccessor> makeFSInputAccessor(
const CanonPath & root,
std::optional<std::set<CanonPath>> && allowedPaths = {},
MakeNotAllowedError && makeNotAllowedError = {});
ref<FSInputAccessor> makeStorePathAccessor(
ref<Store> store,
const StorePath & storePath,
MakeNotAllowedError && makeNotAllowedError = {});
SourcePath getUnfilteredRootPath(CanonPath path);
}

View file

@ -46,7 +46,7 @@ bool touchCacheFile(const Path & path, time_t touch_time)
Path getCachePath(std::string_view key) Path getCachePath(std::string_view key)
{ {
return getCacheDir() + "/nix/gitv3/" + return getCacheDir() + "/nix/gitv3/" +
hashString(htSHA256, key).to_string(Base32, false); hashString(htSHA256, key).to_string(HashFormat::Base32, false);
} }
// Returns the name of the HEAD branch. // Returns the name of the HEAD branch.
@ -322,15 +322,6 @@ struct GitInputScheme : InputScheme
return url; return url;
} }
bool hasAllInfo(const Input & input) const override
{
bool maybeDirty = !input.getRef();
bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
return
maybeGetIntAttr(input.attrs, "lastModified")
&& (shallow || maybeDirty || maybeGetIntAttr(input.attrs, "revCount"));
}
Input applyOverrides( Input applyOverrides(
const Input & input, const Input & input,
std::optional<std::string> ref, std::optional<std::string> ref,
@ -418,7 +409,7 @@ struct GitInputScheme : InputScheme
auto checkHashType = [&](const std::optional<Hash> & hash) auto checkHashType = [&](const std::optional<Hash> & hash)
{ {
if (hash.has_value() && !(hash->type == htSHA1 || hash->type == htSHA256)) if (hash.has_value() && !(hash->type == htSHA1 || hash->type == htSHA256))
throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(Base16, true)); throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(HashFormat::Base16, true));
}; };
auto getLockedAttrs = [&]() auto getLockedAttrs = [&]()

View file

@ -7,6 +7,7 @@
#include "git.hh" #include "git.hh"
#include "fetchers.hh" #include "fetchers.hh"
#include "fetch-settings.hh" #include "fetch-settings.hh"
#include "tarball.hh"
#include <optional> #include <optional>
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
@ -125,18 +126,13 @@ struct GitArchiveInputScheme : InputScheme
auto path = owner + "/" + repo; auto path = owner + "/" + repo;
assert(!(ref && rev)); assert(!(ref && rev));
if (ref) path += "/" + *ref; if (ref) path += "/" + *ref;
if (rev) path += "/" + rev->to_string(Base16, false); if (rev) path += "/" + rev->to_string(HashFormat::Base16, false);
return ParsedURL { return ParsedURL {
.scheme = type(), .scheme = type(),
.path = path, .path = path,
}; };
} }
bool hasAllInfo(const Input & input) const override
{
return input.getRev() && maybeGetIntAttr(input.attrs, "lastModified");
}
Input applyOverrides( Input applyOverrides(
const Input & _input, const Input & _input,
std::optional<std::string> ref, std::optional<std::string> ref,
@ -218,10 +214,15 @@ struct GitArchiveInputScheme : InputScheme
{"rev", rev->gitRev()}, {"rev", rev->gitRev()},
{"lastModified", uint64_t(result.lastModified)} {"lastModified", uint64_t(result.lastModified)}
}, },
result.tree.storePath, result.storePath,
true); true);
return {result.tree.storePath, input}; return {result.storePath, input};
}
std::optional<ExperimentalFeature> experimentalFeature() override
{
return Xp::Flakes;
} }
}; };
@ -291,7 +292,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
: "https://api.%s/repos/%s/%s/tarball/%s"; : "https://api.%s/repos/%s/%s/tarball/%s";
const auto url = fmt(urlFmt, host, getOwner(input), getRepo(input), const auto url = fmt(urlFmt, host, getOwner(input), getRepo(input),
input.getRev()->to_string(Base16, false)); input.getRev()->to_string(HashFormat::Base16, false));
return DownloadUrl { url, headers }; return DownloadUrl { url, headers };
} }
@ -357,7 +358,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s", auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s",
host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
input.getRev()->to_string(Base16, false)); input.getRev()->to_string(HashFormat::Base16, false));
Headers headers = makeHeadersWithAuthTokens(host); Headers headers = makeHeadersWithAuthTokens(host);
return DownloadUrl { url, headers }; return DownloadUrl { url, headers };
@ -444,7 +445,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht"); auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht");
auto url = fmt("https://%s/%s/%s/archive/%s.tar.gz", auto url = fmt("https://%s/%s/%s/archive/%s.tar.gz",
host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
input.getRev()->to_string(Base16, false)); input.getRev()->to_string(HashFormat::Base16, false));
Headers headers = makeHeadersWithAuthTokens(host); Headers headers = makeHeadersWithAuthTokens(host);
return DownloadUrl { url, headers }; return DownloadUrl { url, headers };

View file

@ -41,7 +41,6 @@ struct IndirectInputScheme : InputScheme
// FIXME: forbid query params? // FIXME: forbid query params?
Input input; Input input;
input.direct = false;
input.attrs.insert_or_assign("type", "indirect"); input.attrs.insert_or_assign("type", "indirect");
input.attrs.insert_or_assign("id", id); input.attrs.insert_or_assign("id", id);
if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); if (rev) input.attrs.insert_or_assign("rev", rev->gitRev());
@ -63,7 +62,6 @@ struct IndirectInputScheme : InputScheme
throw BadURL("'%s' is not a valid flake ID", id); throw BadURL("'%s' is not a valid flake ID", id);
Input input; Input input;
input.direct = false;
input.attrs = attrs; input.attrs = attrs;
return input; return input;
} }
@ -78,11 +76,6 @@ struct IndirectInputScheme : InputScheme
return url; return url;
} }
bool hasAllInfo(const Input & input) const override
{
return false;
}
Input applyOverrides( Input applyOverrides(
const Input & _input, const Input & _input,
std::optional<std::string> ref, std::optional<std::string> ref,
@ -103,6 +96,9 @@ struct IndirectInputScheme : InputScheme
{ {
return Xp::Flakes; return Xp::Flakes;
} }
bool isDirect(const Input & input) const override
{ return false; }
}; };
static auto rIndirectInputScheme = OnStartup([] { registerInputScheme(std::make_unique<IndirectInputScheme>()); }); static auto rIndirectInputScheme = OnStartup([] { registerInputScheme(std::make_unique<IndirectInputScheme>()); });

View file

@ -3,12 +3,52 @@
namespace nix { namespace nix {
StorePath InputAccessor::fetchToStore(
ref<Store> store,
const CanonPath & path,
std::string_view name,
FileIngestionMethod method,
PathFilter * filter,
RepairFlag repair)
{
Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", showPath(path)));
auto source = sinkToSource([&](Sink & sink) {
if (method == FileIngestionMethod::Recursive)
dumpPath(path, sink, filter ? *filter : defaultPathFilter);
else
readFile(path, sink);
});
auto storePath =
settings.readOnlyMode
? store->computeStorePathFromDump(*source, name, method, htSHA256).first
: store->addToStoreFromDump(*source, name, method, htSHA256, repair);
return storePath;
}
SourcePath InputAccessor::root()
{
return {ref(shared_from_this()), CanonPath::root};
}
std::ostream & operator << (std::ostream & str, const SourcePath & path) std::ostream & operator << (std::ostream & str, const SourcePath & path)
{ {
str << path.to_string(); str << path.to_string();
return str; return str;
} }
StorePath SourcePath::fetchToStore(
ref<Store> store,
std::string_view name,
FileIngestionMethod method,
PathFilter * filter,
RepairFlag repair) const
{
return accessor->fetchToStore(store, path, name, method, filter, repair);
}
std::string_view SourcePath::baseName() const std::string_view SourcePath::baseName() const
{ {
return path.baseName().value_or("source"); return path.baseName().value_or("source");
@ -18,60 +58,12 @@ SourcePath SourcePath::parent() const
{ {
auto p = path.parent(); auto p = path.parent();
assert(p); assert(p);
return std::move(*p); return {accessor, std::move(*p)};
}
InputAccessor::Stat SourcePath::lstat() const
{
auto st = nix::lstat(path.abs());
return InputAccessor::Stat {
.type =
S_ISREG(st.st_mode) ? InputAccessor::tRegular :
S_ISDIR(st.st_mode) ? InputAccessor::tDirectory :
S_ISLNK(st.st_mode) ? InputAccessor::tSymlink :
InputAccessor::tMisc,
.isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR
};
}
std::optional<InputAccessor::Stat> SourcePath::maybeLstat() const
{
// FIXME: merge these into one operation.
if (!pathExists())
return {};
return lstat();
}
InputAccessor::DirEntries SourcePath::readDirectory() const
{
InputAccessor::DirEntries res;
for (auto & entry : nix::readDirectory(path.abs())) {
std::optional<InputAccessor::Type> type;
switch (entry.type) {
case DT_REG: type = InputAccessor::Type::tRegular; break;
case DT_LNK: type = InputAccessor::Type::tSymlink; break;
case DT_DIR: type = InputAccessor::Type::tDirectory; break;
}
res.emplace(entry.name, type);
}
return res;
}
StorePath SourcePath::fetchToStore(
ref<Store> store,
std::string_view name,
PathFilter * filter,
RepairFlag repair) const
{
return
settings.readOnlyMode
? store->computeStorePathForPath(name, path.abs(), FileIngestionMethod::Recursive, htSHA256, filter ? *filter : defaultPathFilter).first
: store->addToStore(name, path.abs(), FileIngestionMethod::Recursive, htSHA256, filter ? *filter : defaultPathFilter, repair);
} }
SourcePath SourcePath::resolveSymlinks() const SourcePath SourcePath::resolveSymlinks() const
{ {
SourcePath res(CanonPath::root); auto res = accessor->root();
int linksAllowed = 1024; int linksAllowed = 1024;

View file

@ -1,40 +1,39 @@
#pragma once #pragma once
#include "source-accessor.hh"
#include "ref.hh" #include "ref.hh"
#include "types.hh" #include "types.hh"
#include "archive.hh"
#include "canon-path.hh"
#include "repair-flag.hh" #include "repair-flag.hh"
#include "content-address.hh"
namespace nix { namespace nix {
MakeError(RestrictedPathError, Error);
struct SourcePath;
class StorePath; class StorePath;
class Store; class Store;
struct InputAccessor struct InputAccessor : SourceAccessor, std::enable_shared_from_this<InputAccessor>
{ {
enum Type {
tRegular, tSymlink, tDirectory,
/** /**
Any other node types that may be encountered on the file system, such as device nodes, sockets, named pipe, and possibly even more exotic things. * Return the maximum last-modified time of the files in this
* tree, if available.
Responsible for `"unknown"` from `builtins.readFileType "/dev/null"`.
Unlike `DT_UNKNOWN`, this must not be used for deferring the lookup of types.
*/ */
tMisc virtual std::optional<time_t> getLastModified()
};
struct Stat
{ {
Type type = tMisc; return std::nullopt;
//uint64_t fileSize = 0; // regular files only }
bool isExecutable = false; // regular files only
};
typedef std::optional<Type> DirEntry; StorePath fetchToStore(
ref<Store> store,
const CanonPath & path,
std::string_view name = "source",
FileIngestionMethod method = FileIngestionMethod::Recursive,
PathFilter * filter = nullptr,
RepairFlag repair = NoRepair);
typedef std::map<std::string, DirEntry> DirEntries; SourcePath root();
}; };
/** /**
@ -45,12 +44,9 @@ struct InputAccessor
*/ */
struct SourcePath struct SourcePath
{ {
ref<InputAccessor> accessor;
CanonPath path; CanonPath path;
SourcePath(CanonPath path)
: path(std::move(path))
{ }
std::string_view baseName() const; std::string_view baseName() const;
/** /**
@ -64,39 +60,42 @@ struct SourcePath
* return its contents; otherwise throw an error. * return its contents; otherwise throw an error.
*/ */
std::string readFile() const std::string readFile() const
{ return nix::readFile(path.abs()); } { return accessor->readFile(path); }
/** /**
* Return whether this `SourcePath` denotes a file (of any type) * Return whether this `SourcePath` denotes a file (of any type)
* that exists * that exists
*/ */
bool pathExists() const bool pathExists() const
{ return nix::pathExists(path.abs()); } { return accessor->pathExists(path); }
/** /**
* Return stats about this `SourcePath`, or throw an exception if * Return stats about this `SourcePath`, or throw an exception if
* it doesn't exist. * it doesn't exist.
*/ */
InputAccessor::Stat lstat() const; InputAccessor::Stat lstat() const
{ return accessor->lstat(path); }
/** /**
* Return stats about this `SourcePath`, or std::nullopt if it * Return stats about this `SourcePath`, or std::nullopt if it
* doesn't exist. * doesn't exist.
*/ */
std::optional<InputAccessor::Stat> maybeLstat() const; std::optional<InputAccessor::Stat> maybeLstat() const
{ return accessor->maybeLstat(path); }
/** /**
* If this `SourcePath` denotes a directory (not a symlink), * If this `SourcePath` denotes a directory (not a symlink),
* return its directory entries; otherwise throw an error. * return its directory entries; otherwise throw an error.
*/ */
InputAccessor::DirEntries readDirectory() const; InputAccessor::DirEntries readDirectory() const
{ return accessor->readDirectory(path); }
/** /**
* If this `SourcePath` denotes a symlink, return its target; * If this `SourcePath` denotes a symlink, return its target;
* otherwise throw an error. * otherwise throw an error.
*/ */
std::string readLink() const std::string readLink() const
{ return nix::readLink(path.abs()); } { return accessor->readLink(path); }
/** /**
* Dump this `SourcePath` to `sink` as a NAR archive. * Dump this `SourcePath` to `sink` as a NAR archive.
@ -104,7 +103,7 @@ struct SourcePath
void dumpPath( void dumpPath(
Sink & sink, Sink & sink,
PathFilter & filter = defaultPathFilter) const PathFilter & filter = defaultPathFilter) const
{ return nix::dumpPath(path.abs(), sink, filter); } { return accessor->dumpPath(path, sink, filter); }
/** /**
* Copy this `SourcePath` to the Nix store. * Copy this `SourcePath` to the Nix store.
@ -112,6 +111,7 @@ struct SourcePath
StorePath fetchToStore( StorePath fetchToStore(
ref<Store> store, ref<Store> store,
std::string_view name = "source", std::string_view name = "source",
FileIngestionMethod method = FileIngestionMethod::Recursive,
PathFilter * filter = nullptr, PathFilter * filter = nullptr,
RepairFlag repair = NoRepair) const; RepairFlag repair = NoRepair) const;
@ -120,7 +120,7 @@ struct SourcePath
* it has a physical location. * it has a physical location.
*/ */
std::optional<CanonPath> getPhysicalPath() const std::optional<CanonPath> getPhysicalPath() const
{ return path; } { return accessor->getPhysicalPath(path); }
std::string to_string() const std::string to_string() const
{ return path.abs(); } { return path.abs(); }
@ -129,7 +129,7 @@ struct SourcePath
* Append a `CanonPath` to this path. * Append a `CanonPath` to this path.
*/ */
SourcePath operator + (const CanonPath & x) const SourcePath operator + (const CanonPath & x) const
{ return {path + x}; } { return {accessor, path + x}; }
/** /**
* Append a single component `c` to this path. `c` must not * Append a single component `c` to this path. `c` must not
@ -137,21 +137,21 @@ struct SourcePath
* and `c`. * and `c`.
*/ */
SourcePath operator + (std::string_view c) const SourcePath operator + (std::string_view c) const
{ return {path + c}; } { return {accessor, path + c}; }
bool operator == (const SourcePath & x) const bool operator == (const SourcePath & x) const
{ {
return path == x.path; return std::tie(accessor, path) == std::tie(x.accessor, x.path);
} }
bool operator != (const SourcePath & x) const bool operator != (const SourcePath & x) const
{ {
return path != x.path; return std::tie(accessor, path) != std::tie(x.accessor, x.path);
} }
bool operator < (const SourcePath & x) const bool operator < (const SourcePath & x) const
{ {
return path < x.path; return std::tie(accessor, path) < std::tie(x.accessor, x.path);
} }
/** /**

View file

@ -0,0 +1,54 @@
#include "memory-input-accessor.hh"
namespace nix {
struct MemoryInputAccessorImpl : MemoryInputAccessor
{
std::map<CanonPath, std::string> files;
std::string readFile(const CanonPath & path) override
{
auto i = files.find(path);
if (i == files.end())
throw Error("file '%s' does not exist", path);
return i->second;
}
bool pathExists(const CanonPath & path) override
{
auto i = files.find(path);
return i != files.end();
}
Stat lstat(const CanonPath & path) override
{
auto i = files.find(path);
if (i != files.end())
return Stat { .type = tRegular, .isExecutable = false };
throw Error("file '%s' does not exist", path);
}
DirEntries readDirectory(const CanonPath & path) override
{
return {};
}
std::string readLink(const CanonPath & path) override
{
throw UnimplementedError("MemoryInputAccessor::readLink");
}
SourcePath addFile(CanonPath path, std::string && contents) override
{
files.emplace(path, std::move(contents));
return {ref(shared_from_this()), std::move(path)};
}
};
ref<MemoryInputAccessor> makeMemoryInputAccessor()
{
return make_ref<MemoryInputAccessorImpl>();
}
}

View file

@ -0,0 +1,15 @@
#include "input-accessor.hh"
namespace nix {
/**
* An input accessor for an in-memory file system.
*/
struct MemoryInputAccessor : InputAccessor
{
virtual SourcePath addFile(CanonPath path, std::string && contents) = 0;
};
ref<MemoryInputAccessor> makeMemoryInputAccessor();
}

View file

@ -98,13 +98,6 @@ struct MercurialInputScheme : InputScheme
return url; return url;
} }
bool hasAllInfo(const Input & input) const override
{
// FIXME: ugly, need to distinguish between dirty and clean
// default trees.
return input.getRef() == "default" || maybeGetIntAttr(input.attrs, "revCount");
}
Input applyOverrides( Input applyOverrides(
const Input & input, const Input & input,
std::optional<std::string> ref, std::optional<std::string> ref,
@ -206,7 +199,7 @@ struct MercurialInputScheme : InputScheme
auto checkHashType = [&](const std::optional<Hash> & hash) auto checkHashType = [&](const std::optional<Hash> & hash)
{ {
if (hash.has_value() && hash->type != htSHA1) if (hash.has_value() && hash->type != htSHA1)
throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", hash->to_string(Base16, true)); throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", hash->to_string(HashFormat::Base16, true));
}; };
@ -252,7 +245,7 @@ struct MercurialInputScheme : InputScheme
} }
} }
Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(Base32, false)); Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(HashFormat::Base32, false));
/* If this is a commit hash that we already have, we don't /* If this is a commit hash that we already have, we don't
have to pull again. */ have to pull again. */

View file

@ -66,11 +66,6 @@ struct PathInputScheme : InputScheme
}; };
} }
bool hasAllInfo(const Input & input) const override
{
return true;
}
std::optional<Path> getSourcePath(const Input & input) override std::optional<Path> getSourcePath(const Input & input) override
{ {
return getStrAttr(input.attrs, "path"); return getStrAttr(input.attrs, "path");
@ -125,6 +120,11 @@ struct PathInputScheme : InputScheme
return {std::move(*storePath), input}; return {std::move(*storePath), input};
} }
std::optional<ExperimentalFeature> experimentalFeature() override
{
return Xp::Flakes;
}
}; };
static auto rPathInputScheme = OnStartup([] { registerInputScheme(std::make_unique<PathInputScheme>()); }); static auto rPathInputScheme = OnStartup([] { registerInputScheme(std::make_unique<PathInputScheme>()); });

View file

@ -1,5 +1,5 @@
#include "registry.hh" #include "registry.hh"
#include "fetchers.hh" #include "tarball.hh"
#include "util.hh" #include "util.hh"
#include "globals.hh" #include "globals.hh"
#include "store-api.hh" #include "store-api.hh"

View file

@ -1,3 +1,4 @@
#include "tarball.hh"
#include "fetchers.hh" #include "fetchers.hh"
#include "cache.hh" #include "cache.hh"
#include "filetransfer.hh" #include "filetransfer.hh"
@ -133,7 +134,7 @@ DownloadTarballResult downloadTarball(
if (cached && !cached->expired) if (cached && !cached->expired)
return { return {
.tree = Tree { .actualPath = store->toRealPath(cached->storePath), .storePath = std::move(cached->storePath) }, .storePath = std::move(cached->storePath),
.lastModified = (time_t) getIntAttr(cached->infoAttrs, "lastModified"), .lastModified = (time_t) getIntAttr(cached->infoAttrs, "lastModified"),
.immutableUrl = maybeGetStrAttr(cached->infoAttrs, "immutableUrl"), .immutableUrl = maybeGetStrAttr(cached->infoAttrs, "immutableUrl"),
}; };
@ -174,7 +175,7 @@ DownloadTarballResult downloadTarball(
locked); locked);
return { return {
.tree = Tree { .actualPath = store->toRealPath(*unpackedStorePath), .storePath = std::move(*unpackedStorePath) }, .storePath = std::move(*unpackedStorePath),
.lastModified = lastModified, .lastModified = lastModified,
.immutableUrl = res.immutableUrl, .immutableUrl = res.immutableUrl,
}; };
@ -250,15 +251,9 @@ struct CurlInputScheme : InputScheme
// NAR hashes are preferred over file hashes since tar/zip // NAR hashes are preferred over file hashes since tar/zip
// files don't have a canonical representation. // files don't have a canonical representation.
if (auto narHash = input.getNarHash()) if (auto narHash = input.getNarHash())
url.query.insert_or_assign("narHash", narHash->to_string(SRI, true)); url.query.insert_or_assign("narHash", narHash->to_string(HashFormat::SRI, true));
return url; return url;
} }
bool hasAllInfo(const Input & input) const override
{
return true;
}
}; };
struct FileInputScheme : CurlInputScheme struct FileInputScheme : CurlInputScheme
@ -313,7 +308,7 @@ struct TarballInputScheme : CurlInputScheme
if (result.lastModified && !input.attrs.contains("lastModified")) if (result.lastModified && !input.attrs.contains("lastModified"))
input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified)); input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified));
return {result.tree.storePath, std::move(input)}; return {result.storePath, std::move(input)};
} }
}; };

View file

@ -0,0 +1,43 @@
#pragma once
#include "types.hh"
#include "path.hh"
#include <optional>
namespace nix {
class Store;
}
namespace nix::fetchers {
struct DownloadFileResult
{
StorePath storePath;
std::string etag;
std::string effectiveUrl;
std::optional<std::string> immutableUrl;
};
DownloadFileResult downloadFile(
ref<Store> store,
const std::string & url,
const std::string & name,
bool locked,
const Headers & headers = {});
struct DownloadTarballResult
{
StorePath storePath;
time_t lastModified;
std::optional<std::string> immutableUrl;
};
DownloadTarballResult downloadTarball(
ref<Store> store,
const std::string & url,
const std::string & name,
bool locked,
const Headers & headers = {});
}

View file

@ -1,4 +1,5 @@
#include "common-args.hh" #include "common-args.hh"
#include "args/root.hh"
#include "globals.hh" #include "globals.hh"
#include "loggers.hh" #include "loggers.hh"
@ -34,21 +35,21 @@ MixCommonArgs::MixCommonArgs(const std::string & programName)
.description = "Set the Nix configuration setting *name* to *value* (overriding `nix.conf`).", .description = "Set the Nix configuration setting *name* to *value* (overriding `nix.conf`).",
.category = miscCategory, .category = miscCategory,
.labels = {"name", "value"}, .labels = {"name", "value"},
.handler = {[](std::string name, std::string value) { .handler = {[this](std::string name, std::string value) {
try { try {
globalConfig.set(name, value); globalConfig.set(name, value);
} catch (UsageError & e) { } catch (UsageError & e) {
if (!completions) if (!getRoot().completions)
warn(e.what()); warn(e.what());
} }
}}, }},
.completer = [](size_t index, std::string_view prefix) { .completer = [](AddCompletions & completions, size_t index, std::string_view prefix) {
if (index == 0) { if (index == 0) {
std::map<std::string, Config::SettingInfo> settings; std::map<std::string, Config::SettingInfo> settings;
globalConfig.getSettings(settings); globalConfig.getSettings(settings);
for (auto & s : settings) for (auto & s : settings)
if (hasPrefix(s.first, prefix)) if (hasPrefix(s.first, prefix))
completions->add(s.first, fmt("Set the `%s` setting.", s.first)); completions.add(s.first, fmt("Set the `%s` setting.", s.first));
} }
} }
}); });

View file

@ -3,6 +3,7 @@
#include "util.hh" #include "util.hh"
#include "args.hh" #include "args.hh"
#include "args/root.hh"
#include "common-args.hh" #include "common-args.hh"
#include "path.hh" #include "path.hh"
#include "derived-path.hh" #include "derived-path.hh"
@ -66,7 +67,7 @@ template<class N> N getIntArg(const std::string & opt,
} }
struct LegacyArgs : public MixCommonArgs struct LegacyArgs : public MixCommonArgs, public RootArgs
{ {
std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg; std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg;

View file

@ -164,7 +164,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
auto [fileHash, fileSize] = fileHashSink.finish(); auto [fileHash, fileSize] = fileHashSink.finish();
narInfo->fileHash = fileHash; narInfo->fileHash = fileHash;
narInfo->fileSize = fileSize; narInfo->fileSize = fileSize;
narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar" narInfo->url = "nar/" + narInfo->fileHash->to_string(HashFormat::Base32, false) + ".nar"
+ (compression == "xz" ? ".xz" : + (compression == "xz" ? ".xz" :
compression == "bzip2" ? ".bz2" : compression == "bzip2" ? ".bz2" :
compression == "zstd" ? ".zst" : compression == "zstd" ? ".zst" :

View file

@ -561,7 +561,7 @@ void DerivationGoal::inputsRealised()
attempt = fullDrv.tryResolve(worker.store); attempt = fullDrv.tryResolve(worker.store);
} }
assert(attempt); assert(attempt);
Derivation drvResolved { *std::move(attempt) }; Derivation drvResolved { std::move(*attempt) };
auto pathResolved = writeDerivation(worker.store, drvResolved); auto pathResolved = writeDerivation(worker.store, drvResolved);

View file

@ -227,7 +227,7 @@ void LocalDerivationGoal::tryLocalBuild()
if (!buildUser) { if (!buildUser) {
if (!actLock) if (!actLock)
actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting, actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); fmt("waiting for a free build user ID for '%s'", yellowtxt(worker.store.printStorePath(drvPath))));
worker.waitForAWhile(shared_from_this()); worker.waitForAWhile(shared_from_this());
return; return;
} }
@ -386,27 +386,27 @@ void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck()
cleanupPostOutputsRegisteredModeCheck(); cleanupPostOutputsRegisteredModeCheck();
} }
#if __linux__ #if __linux__
static void linkOrCopy(const Path & from, const Path & to) static void doBind(const Path & source, const Path & target, bool optional = false) {
{ debug("bind mounting '%1%' to '%2%'", source, target);
if (link(from.c_str(), to.c_str()) == -1) { struct stat st;
/* Hard-linking fails if we exceed the maximum link count on a if (stat(source.c_str(), &st) == -1) {
file (e.g. 32000 of ext3), which is quite possible after a if (optional && errno == ENOENT)
'nix-store --optimise'. FIXME: actually, why don't we just return;
bind-mount in this case? else
throw SysError("getting attributes of path '%1%'", source);
It can also fail with EPERM in BeegFS v7 and earlier versions
or fail with EXDEV in OpenAFS
which don't allow hard-links to other directories */
if (errno != EMLINK && errno != EPERM && errno != EXDEV)
throw SysError("linking '%s' to '%s'", to, from);
copyPath(from, to);
} }
} if (S_ISDIR(st.st_mode))
createDirs(target);
else {
createDirs(dirOf(target));
writeFile(target, "");
}
if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1)
throw SysError("bind mount from '%1%' to '%2%' failed", source, target);
};
#endif #endif
void LocalDerivationGoal::startBuilder() void LocalDerivationGoal::startBuilder()
{ {
if ((buildUser && buildUser->getUIDCount() != 1) if ((buildUser && buildUser->getUIDCount() != 1)
@ -581,7 +581,7 @@ void LocalDerivationGoal::startBuilder()
/* Allow a user-configurable set of directories from the /* Allow a user-configurable set of directories from the
host file system. */ host file system. */
dirsInChroot.clear(); pathsInChroot.clear();
for (auto i : settings.sandboxPaths.get()) { for (auto i : settings.sandboxPaths.get()) {
if (i.empty()) continue; if (i.empty()) continue;
@ -592,19 +592,19 @@ void LocalDerivationGoal::startBuilder()
} }
size_t p = i.find('='); size_t p = i.find('=');
if (p == std::string::npos) if (p == std::string::npos)
dirsInChroot[i] = {i, optional}; pathsInChroot[i] = {i, optional};
else else
dirsInChroot[i.substr(0, p)] = {i.substr(p + 1), optional}; pathsInChroot[i.substr(0, p)] = {i.substr(p + 1), optional};
} }
if (hasPrefix(worker.store.storeDir, tmpDirInSandbox)) if (hasPrefix(worker.store.storeDir, tmpDirInSandbox))
{ {
throw Error("`sandbox-build-dir` must not contain the storeDir"); throw Error("`sandbox-build-dir` must not contain the storeDir");
} }
dirsInChroot[tmpDirInSandbox] = tmpDir; pathsInChroot[tmpDirInSandbox] = tmpDir;
/* Add the closure of store paths to the chroot. */ /* Add the closure of store paths to the chroot. */
StorePathSet closure; StorePathSet closure;
for (auto & i : dirsInChroot) for (auto & i : pathsInChroot)
try { try {
if (worker.store.isInStore(i.second.source)) if (worker.store.isInStore(i.second.source))
worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure); worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure);
@ -615,7 +615,7 @@ void LocalDerivationGoal::startBuilder()
} }
for (auto & i : closure) { for (auto & i : closure) {
auto p = worker.store.printStorePath(i); auto p = worker.store.printStorePath(i);
dirsInChroot.insert_or_assign(p, p); pathsInChroot.insert_or_assign(p, p);
} }
PathSet allowedPaths = settings.allowedImpureHostPrefixes; PathSet allowedPaths = settings.allowedImpureHostPrefixes;
@ -643,7 +643,7 @@ void LocalDerivationGoal::startBuilder()
/* Allow files in __impureHostDeps to be missing; e.g. /* Allow files in __impureHostDeps to be missing; e.g.
macOS 11+ has no /usr/lib/libSystem*.dylib */ macOS 11+ has no /usr/lib/libSystem*.dylib */
dirsInChroot[i] = {i, true}; pathsInChroot[i] = {i, true};
} }
#if __linux__ #if __linux__
@ -711,15 +711,12 @@ void LocalDerivationGoal::startBuilder()
for (auto & i : inputPaths) { for (auto & i : inputPaths) {
auto p = worker.store.printStorePath(i); auto p = worker.store.printStorePath(i);
Path r = worker.store.toRealPath(p); Path r = worker.store.toRealPath(p);
if (S_ISDIR(lstat(r).st_mode)) pathsInChroot.insert_or_assign(p, r);
dirsInChroot.insert_or_assign(p, r);
else
linkOrCopy(r, chrootRootDir + p);
} }
/* If we're repairing, checking or rebuilding part of a /* If we're repairing, checking or rebuilding part of a
multiple-outputs derivation, it's possible that we're multiple-outputs derivation, it's possible that we're
rebuilding a path that is in settings.dirsInChroot rebuilding a path that is in settings.sandbox-paths
(typically the dependencies of /bin/sh). Throw them (typically the dependencies of /bin/sh). Throw them
out. */ out. */
for (auto & i : drv->outputsAndOptPaths(worker.store)) { for (auto & i : drv->outputsAndOptPaths(worker.store)) {
@ -729,7 +726,7 @@ void LocalDerivationGoal::startBuilder()
is already in the sandbox, so we don't need to worry about is already in the sandbox, so we don't need to worry about
removing it. */ removing it. */
if (i.second.second) if (i.second.second)
dirsInChroot.erase(worker.store.printStorePath(*i.second.second)); pathsInChroot.erase(worker.store.printStorePath(*i.second.second));
} }
if (cgroup) { if (cgroup) {
@ -787,9 +784,9 @@ void LocalDerivationGoal::startBuilder()
} else { } else {
auto p = line.find('='); auto p = line.find('=');
if (p == std::string::npos) if (p == std::string::npos)
dirsInChroot[line] = line; pathsInChroot[line] = line;
else else
dirsInChroot[line.substr(0, p)] = line.substr(p + 1); pathsInChroot[line.substr(0, p)] = line.substr(p + 1);
} }
} }
} }
@ -1066,7 +1063,7 @@ void LocalDerivationGoal::initTmpDir() {
env[i.first] = i.second; env[i.first] = i.second;
} else { } else {
auto hash = hashString(htSHA256, i.first); auto hash = hashString(htSHA256, i.first);
std::string fn = ".attr-" + hash.to_string(Base32, false); std::string fn = ".attr-" + hash.to_string(HashFormat::Base32, false);
Path p = tmpDir + "/" + fn; Path p = tmpDir + "/" + fn;
writeFile(p, rewriteStrings(i.second, inputRewrites)); writeFile(p, rewriteStrings(i.second, inputRewrites));
chownToBuilder(p); chownToBuilder(p);
@ -1565,15 +1562,12 @@ void LocalDerivationGoal::addDependency(const StorePath & path)
Path source = worker.store.Store::toRealPath(path); Path source = worker.store.Store::toRealPath(path);
Path target = chrootRootDir + worker.store.printStorePath(path); Path target = chrootRootDir + worker.store.printStorePath(path);
debug("bind-mounting %s -> %s", target, source);
if (pathExists(target)) if (pathExists(target))
// There is a similar debug message in doBind, so only run it in this block to not have double messages.
debug("bind-mounting %s -> %s", target, source);
throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path)); throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path));
auto st = lstat(source);
if (S_ISDIR(st.st_mode)) {
/* Bind-mount the path into the sandbox. This requires /* Bind-mount the path into the sandbox. This requires
entering its mount namespace, which is not possible entering its mount namespace, which is not possible
in multithreaded programs. So we do this in a in multithreaded programs. So we do this in a
@ -1586,10 +1580,7 @@ void LocalDerivationGoal::addDependency(const StorePath & path)
if (setns(sandboxMountNamespace.get(), 0) == -1) if (setns(sandboxMountNamespace.get(), 0) == -1)
throw SysError("entering sandbox mount namespace"); throw SysError("entering sandbox mount namespace");
createDirs(target); doBind(source, target);
if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1)
throw SysError("bind mount from '%s' to '%s' failed", source, target);
_exit(0); _exit(0);
})); }));
@ -1598,9 +1589,6 @@ void LocalDerivationGoal::addDependency(const StorePath & path)
if (status != 0) if (status != 0)
throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path)); throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path));
} else
linkOrCopy(source, target);
#else #else
throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox", throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox",
worker.store.printStorePath(path)); worker.store.printStorePath(path));
@ -1789,7 +1777,7 @@ void LocalDerivationGoal::runChild()
/* Set up a nearly empty /dev, unless the user asked to /* Set up a nearly empty /dev, unless the user asked to
bind-mount the host /dev. */ bind-mount the host /dev. */
Strings ss; Strings ss;
if (dirsInChroot.find("/dev") == dirsInChroot.end()) { if (pathsInChroot.find("/dev") == pathsInChroot.end()) {
createDirs(chrootRootDir + "/dev/shm"); createDirs(chrootRootDir + "/dev/shm");
createDirs(chrootRootDir + "/dev/pts"); createDirs(chrootRootDir + "/dev/pts");
ss.push_back("/dev/full"); ss.push_back("/dev/full");
@ -1824,34 +1812,15 @@ void LocalDerivationGoal::runChild()
ss.push_back(path); ss.push_back(path);
if (settings.caFile != "") if (settings.caFile != "")
dirsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", settings.caFile, true); pathsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", settings.caFile, true);
} }
for (auto & i : ss) dirsInChroot.emplace(i, i); for (auto & i : ss) pathsInChroot.emplace(i, i);
/* Bind-mount all the directories from the "host" /* Bind-mount all the directories from the "host"
filesystem that we want in the chroot filesystem that we want in the chroot
environment. */ environment. */
auto doBind = [&](const Path & source, const Path & target, bool optional = false) { for (auto & i : pathsInChroot) {
debug("bind mounting '%1%' to '%2%'", source, target);
struct stat st;
if (stat(source.c_str(), &st) == -1) {
if (optional && errno == ENOENT)
return;
else
throw SysError("getting attributes of path '%1%'", source);
}
if (S_ISDIR(st.st_mode))
createDirs(target);
else {
createDirs(dirOf(target));
writeFile(target, "");
}
if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1)
throw SysError("bind mount from '%1%' to '%2%' failed", source, target);
};
for (auto & i : dirsInChroot) {
if (i.second.source == "/proc") continue; // backwards compatibility if (i.second.source == "/proc") continue; // backwards compatibility
#if HAVE_EMBEDDED_SANDBOX_SHELL #if HAVE_EMBEDDED_SANDBOX_SHELL
@ -1892,7 +1861,7 @@ void LocalDerivationGoal::runChild()
if /dev/ptx/ptmx exists). */ if /dev/ptx/ptmx exists). */
if (pathExists("/dev/pts/ptmx") && if (pathExists("/dev/pts/ptmx") &&
!pathExists(chrootRootDir + "/dev/ptmx") !pathExists(chrootRootDir + "/dev/ptmx")
&& !dirsInChroot.count("/dev/pts")) && !pathsInChroot.count("/dev/pts"))
{ {
if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0) if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0)
{ {
@ -2027,7 +1996,7 @@ void LocalDerivationGoal::runChild()
/* We build the ancestry before adding all inputPaths to the store because we know they'll /* We build the ancestry before adding all inputPaths to the store because we know they'll
all have the same parents (the store), and there might be lots of inputs. This isn't all have the same parents (the store), and there might be lots of inputs. This isn't
particularly efficient... I doubt it'll be a bottleneck in practice */ particularly efficient... I doubt it'll be a bottleneck in practice */
for (auto & i : dirsInChroot) { for (auto & i : pathsInChroot) {
Path cur = i.first; Path cur = i.first;
while (cur.compare("/") != 0) { while (cur.compare("/") != 0) {
cur = dirOf(cur); cur = dirOf(cur);
@ -2035,7 +2004,7 @@ void LocalDerivationGoal::runChild()
} }
} }
/* And we want the store in there regardless of how empty dirsInChroot. We include the innermost /* And we want the store in there regardless of how empty pathsInChroot. We include the innermost
path component this time, since it's typically /nix/store and we care about that. */ path component this time, since it's typically /nix/store and we care about that. */
Path cur = worker.store.storeDir; Path cur = worker.store.storeDir;
while (cur.compare("/") != 0) { while (cur.compare("/") != 0) {
@ -2046,7 +2015,7 @@ void LocalDerivationGoal::runChild()
/* Add all our input paths to the chroot */ /* Add all our input paths to the chroot */
for (auto & i : inputPaths) { for (auto & i : inputPaths) {
auto p = worker.store.printStorePath(i); auto p = worker.store.printStorePath(i);
dirsInChroot[p] = p; pathsInChroot[p] = p;
} }
/* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */
@ -2077,7 +2046,7 @@ void LocalDerivationGoal::runChild()
without file-write* allowed, access() incorrectly returns EPERM without file-write* allowed, access() incorrectly returns EPERM
*/ */
sandboxProfile += "(allow file-read* file-write* process-exec\n"; sandboxProfile += "(allow file-read* file-write* process-exec\n";
for (auto & i : dirsInChroot) { for (auto & i : pathsInChroot) {
if (i.first != i.second.source) if (i.first != i.second.source)
throw Error( throw Error(
"can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin", "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin",
@ -2521,7 +2490,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
ValidPathInfo newInfo0 { ValidPathInfo newInfo0 {
worker.store, worker.store,
outputPathName(drv->name, outputName), outputPathName(drv->name, outputName),
*std::move(optCA), std::move(*optCA),
Hash::dummy, Hash::dummy,
}; };
if (*scratchPath != newInfo0.path) { if (*scratchPath != newInfo0.path) {
@ -2583,8 +2552,8 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
delayedException = std::make_exception_ptr( delayedException = std::make_exception_ptr(
BuildError("hash mismatch in fixed-output derivation '%s':\n specified: %s\n got: %s", BuildError("hash mismatch in fixed-output derivation '%s':\n specified: %s\n got: %s",
worker.store.printStorePath(drvPath), worker.store.printStorePath(drvPath),
wanted.to_string(SRI, true), wanted.to_string(HashFormat::SRI, true),
got.to_string(SRI, true))); got.to_string(HashFormat::SRI, true)));
} }
if (!newInfo0.references.empty()) if (!newInfo0.references.empty())
delayedException = std::make_exception_ptr( delayedException = std::make_exception_ptr(

View file

@ -86,8 +86,8 @@ struct LocalDerivationGoal : public DerivationGoal
: source(source), optional(optional) : source(source), optional(optional)
{ } { }
}; };
typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path typedef map<Path, ChrootPath> PathsInChroot; // maps target path to source path
DirsInChroot dirsInChroot; PathsInChroot pathsInChroot;
typedef map<std::string, std::string> Environment; typedef map<std::string, std::string> Environment;
Environment env; Environment env;
@ -120,14 +120,6 @@ struct LocalDerivationGoal : public DerivationGoal
*/ */
OutputPathMap scratchOutputs; OutputPathMap scratchOutputs;
/**
* Path registration info from the previous round, if we're
* building multiple times. Since this contains the hash, it
* allows us to compare whether two rounds produced the same
* result.
*/
std::map<Path, ValidPathInfo> prevInfos;
uid_t sandboxUid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); } uid_t sandboxUid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); }
gid_t sandboxGid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 100 : 0) : buildUser->getGID(); } gid_t sandboxGid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 100 : 0) : buildUser->getGID(); }

View file

@ -174,15 +174,19 @@ void builtinBuildenv(const BasicDerivation & drv)
/* Convert the stuff we get from the environment back into a /* Convert the stuff we get from the environment back into a
* coherent data type. */ * coherent data type. */
Packages pkgs; Packages pkgs;
{
auto derivations = tokenizeString<Strings>(getAttr("derivations")); auto derivations = tokenizeString<Strings>(getAttr("derivations"));
while (!derivations.empty()) {
auto itemIt = derivations.begin();
while (itemIt != derivations.end()) {
/* !!! We're trusting the caller to structure derivations env var correctly */ /* !!! We're trusting the caller to structure derivations env var correctly */
auto active = derivations.front(); derivations.pop_front(); const bool active = "false" != *itemIt++;
auto priority = stoi(derivations.front()); derivations.pop_front(); const int priority = stoi(*itemIt++);
auto outputs = stoi(derivations.front()); derivations.pop_front(); const size_t outputs = stoul(*itemIt++);
for (auto n = 0; n < outputs; n++) {
auto path = derivations.front(); derivations.pop_front(); for (size_t n {0}; n < outputs; n++) {
pkgs.emplace_back(path, active != "false", priority); pkgs.emplace_back(std::move(*itemIt++), active, priority);
}
} }
} }

View file

@ -65,7 +65,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
std::optional<HashType> ht = parseHashTypeOpt(getAttr("outputHashAlgo")); std::optional<HashType> ht = parseHashTypeOpt(getAttr("outputHashAlgo"));
Hash h = newHashAllowEmpty(getAttr("outputHash"), ht); Hash h = newHashAllowEmpty(getAttr("outputHash"), ht);
fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false)); fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(HashFormat::Base16, false));
return; return;
} catch (Error & e) { } catch (Error & e) {
debug(e.what()); debug(e.what());

View file

@ -29,12 +29,13 @@ std::string ContentAddressMethod::renderPrefix() const
ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m) ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m)
{ {
ContentAddressMethod method = FileIngestionMethod::Flat; if (splitPrefix(m, "r:")) {
if (splitPrefix(m, "r:")) return FileIngestionMethod::Recursive;
method = FileIngestionMethod::Recursive; }
else if (splitPrefix(m, "text:")) else if (splitPrefix(m, "text:")) {
method = TextIngestionMethod {}; return TextIngestionMethod {};
return method; }
return FileIngestionMethod::Flat;
} }
std::string ContentAddressMethod::render(HashType ht) const std::string ContentAddressMethod::render(HashType ht) const
@ -60,7 +61,7 @@ std::string ContentAddress::render() const
+ makeFileIngestionPrefix(method); + makeFileIngestionPrefix(method);
}, },
}, method.raw) }, method.raw)
+ this->hash.to_string(Base32, true); + this->hash.to_string(HashFormat::Base32, true);
} }
/** /**
@ -83,7 +84,7 @@ static std::pair<ContentAddressMethod, HashType> parseContentAddressMethodPrefix
if (!hashTypeRaw) if (!hashTypeRaw)
throw UsageError("content address hash must be in form '<algo>:<hash>', but found: %s", wholeInput); throw UsageError("content address hash must be in form '<algo>:<hash>', but found: %s", wholeInput);
HashType hashType = parseHashType(*hashTypeRaw); HashType hashType = parseHashType(*hashTypeRaw);
return std::move(hashType); return hashType;
}; };
// Switch on prefix // Switch on prefix

View file

@ -45,9 +45,9 @@ struct TunnelLogger : public Logger
Sync<State> state_; Sync<State> state_;
unsigned int clientVersion; WorkerProto::Version clientVersion;
TunnelLogger(FdSink & to, unsigned int clientVersion) TunnelLogger(FdSink & to, WorkerProto::Version clientVersion)
: to(to), clientVersion(clientVersion) { } : to(to), clientVersion(clientVersion) { }
void enqueueMsg(const std::string & s) void enqueueMsg(const std::string & s)
@ -261,24 +261,18 @@ struct ClientSettings
} }
}; };
static std::vector<DerivedPath> readDerivedPaths(Store & store, unsigned int clientVersion, WorkerProto::ReadConn conn)
{
std::vector<DerivedPath> reqs;
if (GET_PROTOCOL_MINOR(clientVersion) >= 30) {
reqs = WorkerProto::Serialise<std::vector<DerivedPath>>::read(store, conn);
} else {
for (auto & s : readStrings<Strings>(conn.from))
reqs.push_back(parsePathWithOutputs(store, s).toDerivedPath());
}
return reqs;
}
static void performOp(TunnelLogger * logger, ref<Store> store, static void performOp(TunnelLogger * logger, ref<Store> store,
TrustedFlag trusted, RecursiveFlag recursive, unsigned int clientVersion, TrustedFlag trusted, RecursiveFlag recursive, WorkerProto::Version clientVersion,
Source & from, BufferedSink & to, WorkerProto::Op op) Source & from, BufferedSink & to, WorkerProto::Op op)
{ {
WorkerProto::ReadConn rconn { .from = from }; WorkerProto::ReadConn rconn {
WorkerProto::WriteConn wconn { .to = to }; .from = from,
.version = clientVersion,
};
WorkerProto::WriteConn wconn {
.to = to,
.version = clientVersion,
};
switch (op) { switch (op) {
@ -334,7 +328,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork(); logger->startWork();
auto hash = store->queryPathInfo(path)->narHash; auto hash = store->queryPathInfo(path)->narHash;
logger->stopWork(); logger->stopWork();
to << hash.to_string(Base16, false); to << hash.to_string(HashFormat::Base16, false);
break; break;
} }
@ -428,7 +422,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}(); }();
logger->stopWork(); logger->stopWork();
pathInfo->write(to, *store, GET_PROTOCOL_MINOR(clientVersion)); WorkerProto::Serialise<ValidPathInfo>::write(*store, wconn, *pathInfo);
} else { } else {
HashType hashAlgo; HashType hashAlgo;
std::string baseName; std::string baseName;
@ -532,7 +526,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
} }
case WorkerProto::Op::BuildPaths: { case WorkerProto::Op::BuildPaths: {
auto drvs = readDerivedPaths(*store, clientVersion, rconn); auto drvs = WorkerProto::Serialise<DerivedPaths>::read(*store, rconn);
BuildMode mode = bmNormal; BuildMode mode = bmNormal;
if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { if (GET_PROTOCOL_MINOR(clientVersion) >= 15) {
mode = (BuildMode) readInt(from); mode = (BuildMode) readInt(from);
@ -557,7 +551,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
} }
case WorkerProto::Op::BuildPathsWithResults: { case WorkerProto::Op::BuildPathsWithResults: {
auto drvs = readDerivedPaths(*store, clientVersion, rconn); auto drvs = WorkerProto::Serialise<DerivedPaths>::read(*store, rconn);
BuildMode mode = bmNormal; BuildMode mode = bmNormal;
mode = (BuildMode) readInt(from); mode = (BuildMode) readInt(from);
@ -641,16 +635,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
auto res = store->buildDerivation(drvPath, drv, buildMode); auto res = store->buildDerivation(drvPath, drv, buildMode);
logger->stopWork(); logger->stopWork();
to << res.status << res.errorMsg; WorkerProto::write(*store, wconn, res);
if (GET_PROTOCOL_MINOR(clientVersion) >= 29) {
to << res.timesBuilt << res.isNonDeterministic << res.startTime << res.stopTime;
}
if (GET_PROTOCOL_MINOR(clientVersion) >= 28) {
DrvOutputs builtOutputs;
for (auto & [output, realisation] : res.builtOutputs)
builtOutputs.insert_or_assign(realisation.id, realisation);
WorkerProto::write(*store, wconn, builtOutputs);
}
break; break;
} }
@ -834,7 +819,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
if (info) { if (info) {
if (GET_PROTOCOL_MINOR(clientVersion) >= 17) if (GET_PROTOCOL_MINOR(clientVersion) >= 17)
to << 1; to << 1;
info->write(to, *store, GET_PROTOCOL_MINOR(clientVersion), false); WorkerProto::write(*store, wconn, static_cast<const UnkeyedValidPathInfo &>(*info));
} else { } else {
assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); assert(GET_PROTOCOL_MINOR(clientVersion) >= 17);
to << 0; to << 0;
@ -932,7 +917,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
} }
case WorkerProto::Op::QueryMissing: { case WorkerProto::Op::QueryMissing: {
auto targets = readDerivedPaths(*store, clientVersion, rconn); auto targets = WorkerProto::Serialise<DerivedPaths>::read(*store, rconn);
logger->startWork(); logger->startWork();
StorePathSet willBuild, willSubstitute, unknown; StorePathSet willBuild, willSubstitute, unknown;
uint64_t downloadSize, narSize; uint64_t downloadSize, narSize;
@ -1017,7 +1002,7 @@ void processConnection(
if (magic != WORKER_MAGIC_1) throw Error("protocol mismatch"); if (magic != WORKER_MAGIC_1) throw Error("protocol mismatch");
to << WORKER_MAGIC_2 << PROTOCOL_VERSION; to << WORKER_MAGIC_2 << PROTOCOL_VERSION;
to.flush(); to.flush();
unsigned int clientVersion = readInt(from); WorkerProto::Version clientVersion = readInt(from);
if (clientVersion < 0x10a) if (clientVersion < 0x10a)
throw Error("the Nix client version is too old"); throw Error("the Nix client version is too old");
@ -1052,7 +1037,10 @@ void processConnection(
auto temp = trusted auto temp = trusted
? store->isTrustedClient() ? store->isTrustedClient()
: std::optional { NotTrusted }; : std::optional { NotTrusted };
WorkerProto::WriteConn wconn { .to = to }; WorkerProto::WriteConn wconn {
.to = to,
.version = clientVersion,
};
WorkerProto::write(*store, wconn, temp); WorkerProto::write(*store, wconn, temp);
} }

View file

@ -542,7 +542,7 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs,
[&](const DerivationOutput::CAFixed & dof) { [&](const DerivationOutput::CAFixed & dof) {
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first))); s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first)));
s += ','; printUnquotedString(s, dof.ca.printMethodAlgo()); s += ','; printUnquotedString(s, dof.ca.printMethodAlgo());
s += ','; printUnquotedString(s, dof.ca.hash.to_string(Base16, false)); s += ','; printUnquotedString(s, dof.ca.hash.to_string(HashFormat::Base16, false));
}, },
[&](const DerivationOutput::CAFloating & dof) { [&](const DerivationOutput::CAFloating & dof) {
s += ','; printUnquotedString(s, ""); s += ','; printUnquotedString(s, "");
@ -775,7 +775,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
auto & dof = std::get<DerivationOutput::CAFixed>(i.second.raw); auto & dof = std::get<DerivationOutput::CAFixed>(i.second.raw);
auto hash = hashString(htSHA256, "fixed:out:" auto hash = hashString(htSHA256, "fixed:out:"
+ dof.ca.printMethodAlgo() + ":" + dof.ca.printMethodAlgo() + ":"
+ dof.ca.hash.to_string(Base16, false) + ":" + dof.ca.hash.to_string(HashFormat::Base16, false) + ":"
+ store.printStorePath(dof.path(store, drv.name, i.first))); + store.printStorePath(dof.path(store, drv.name, i.first)));
outputHashes.insert_or_assign(i.first, std::move(hash)); outputHashes.insert_or_assign(i.first, std::move(hash));
} }
@ -820,7 +820,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
const auto h = get(res.hashes, outputName); const auto h = get(res.hashes, outputName);
if (!h) if (!h)
throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name); throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name);
inputs2[h->to_string(Base16, false)].value.insert(outputName); inputs2[h->to_string(HashFormat::Base16, false)].value.insert(outputName);
} }
} }
@ -925,7 +925,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
[&](const DerivationOutput::CAFixed & dof) { [&](const DerivationOutput::CAFixed & dof) {
out << store.printStorePath(dof.path(store, drv.name, i.first)) out << store.printStorePath(dof.path(store, drv.name, i.first))
<< dof.ca.printMethodAlgo() << dof.ca.printMethodAlgo()
<< dof.ca.hash.to_string(Base16, false); << dof.ca.hash.to_string(HashFormat::Base16, false);
}, },
[&](const DerivationOutput::CAFloating & dof) { [&](const DerivationOutput::CAFloating & dof) {
out << "" out << ""
@ -957,7 +957,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
std::string hashPlaceholder(const OutputNameView outputName) std::string hashPlaceholder(const OutputNameView outputName)
{ {
// FIXME: memoize? // FIXME: memoize?
return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(Base32, false); return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(HashFormat::Base32, false);
} }
@ -1162,7 +1162,7 @@ nlohmann::json DerivationOutput::toJSON(
[&](const DerivationOutput::CAFixed & dof) { [&](const DerivationOutput::CAFixed & dof) {
res["path"] = store.printStorePath(dof.path(store, drvName, outputName)); res["path"] = store.printStorePath(dof.path(store, drvName, outputName));
res["hashAlgo"] = dof.ca.printMethodAlgo(); res["hashAlgo"] = dof.ca.printMethodAlgo();
res["hash"] = dof.ca.hash.to_string(Base16, false); res["hash"] = dof.ca.hash.to_string(HashFormat::Base16, false);
// FIXME print refs? // FIXME print refs?
}, },
[&](const DerivationOutput::CAFloating & dof) { [&](const DerivationOutput::CAFloating & dof) {

View file

@ -5,7 +5,7 @@ namespace nix {
std::string DownstreamPlaceholder::render() const std::string DownstreamPlaceholder::render() const
{ {
return "/" + hash.to_string(Base32, false); return "/" + hash.to_string(HashFormat::Base32, false);
} }
@ -31,7 +31,7 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation(
xpSettings.require(Xp::DynamicDerivations); xpSettings.require(Xp::DynamicDerivations);
auto compressed = compressHash(placeholder.hash, 20); auto compressed = compressHash(placeholder.hash, 20);
auto clearText = "nix-computed-output:" auto clearText = "nix-computed-output:"
+ compressed.to_string(Base32, false) + compressed.to_string(HashFormat::Base32, false)
+ ":" + std::string { outputName }; + ":" + std::string { outputName };
return DownstreamPlaceholder { return DownstreamPlaceholder {
hashString(htSHA256, clearText) hashString(htSHA256, clearText)

View file

@ -41,7 +41,7 @@ void Store::exportPath(const StorePath & path, Sink & sink)
Hash hash = hashSink.currentHash().first; Hash hash = hashSink.currentHash().first;
if (hash != info->narHash && info->narHash != Hash(info->narHash.type)) if (hash != info->narHash && info->narHash != Hash(info->narHash.type))
throw Error("hash of path '%s' has changed from '%s' to '%s'!", throw Error("hash of path '%s' has changed from '%s' to '%s'!",
printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true)); printStorePath(path), info->narHash.to_string(HashFormat::Base32, true), hash.to_string(HashFormat::Base32, true));
teeSink teeSink
<< exportMagic << exportMagic

View file

@ -43,7 +43,7 @@ static void makeSymlink(const Path & link, const Path & target)
void LocalStore::addIndirectRoot(const Path & path) void LocalStore::addIndirectRoot(const Path & path)
{ {
std::string hash = hashString(htSHA1, path).to_string(Base32, false); std::string hash = hashString(htSHA1, path).to_string(HashFormat::Base32, false);
Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", stateDir, gcRootsDir, hash)); Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", stateDir, gcRootsDir, hash));
makeSymlink(realRoot, path); makeSymlink(realRoot, path);
} }

View file

@ -24,6 +24,9 @@
#include "config-impl.hh" #include "config-impl.hh"
#ifdef __APPLE__
#include <sys/sysctl.h>
#endif
namespace nix { namespace nix {
@ -154,6 +157,29 @@ unsigned int Settings::getDefaultCores()
return concurrency; return concurrency;
} }
#if __APPLE__
static bool hasVirt() {
int hasVMM;
int hvSupport;
size_t size;
size = sizeof(hasVMM);
if (sysctlbyname("kern.hv_vmm_present", &hasVMM, &size, NULL, 0) == 0) {
if (hasVMM)
return false;
}
// whether the kernel and hardware supports virt
size = sizeof(hvSupport);
if (sysctlbyname("kern.hv_support", &hvSupport, &size, NULL, 0) == 0) {
return hvSupport == 1;
} else {
return false;
}
}
#endif
StringSet Settings::getDefaultSystemFeatures() StringSet Settings::getDefaultSystemFeatures()
{ {
/* For backwards compatibility, accept some "features" that are /* For backwards compatibility, accept some "features" that are
@ -170,6 +196,11 @@ StringSet Settings::getDefaultSystemFeatures()
features.insert("kvm"); features.insert("kvm");
#endif #endif
#if __APPLE__
if (hasVirt())
features.insert("apple-virt");
#endif
return features; return features;
} }

View file

@ -714,9 +714,13 @@ public:
System features are user-defined, but Nix sets the following defaults: System features are user-defined, but Nix sets the following defaults:
- `apple-virt`
Included on Darwin if virtualization is available.
- `kvm` - `kvm`
Included by default if `/dev/kvm` is accessible. Included on Linux if `/dev/kvm` is accessible.
- `nixos-test`, `benchmark`, `big-parallel` - `nixos-test`, `benchmark`, `big-parallel`

View file

@ -45,7 +45,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
std::unique_ptr<SSHMaster::Connection> sshConn; std::unique_ptr<SSHMaster::Connection> sshConn;
FdSink to; FdSink to;
FdSource from; FdSource from;
int remoteVersion; ServeProto::Version remoteVersion;
bool good = true; bool good = true;
/** /**
@ -60,6 +60,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
{ {
return ServeProto::ReadConn { return ServeProto::ReadConn {
.from = from, .from = from,
.version = remoteVersion,
}; };
} }
@ -75,6 +76,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
{ {
return ServeProto::WriteConn { return ServeProto::WriteConn {
.to = to, .to = to,
.version = remoteVersion,
}; };
} }
}; };
@ -209,7 +211,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
<< ServeProto::Command::AddToStoreNar << ServeProto::Command::AddToStoreNar
<< printStorePath(info.path) << printStorePath(info.path)
<< (info.deriver ? printStorePath(*info.deriver) : "") << (info.deriver ? printStorePath(*info.deriver) : "")
<< info.narHash.to_string(Base16, false); << info.narHash.to_string(HashFormat::Base16, false);
ServeProto::write(*this, *conn, info.references); ServeProto::write(*this, *conn, info.references);
conn->to conn->to
<< info.registrationTime << info.registrationTime
@ -317,20 +319,7 @@ public:
conn->to.flush(); conn->to.flush();
BuildResult status; return ServeProto::Serialise<BuildResult>::read(*this, *conn);
status.status = (BuildResult::Status) readInt(conn->from);
conn->from >> status.errorMsg;
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3)
conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime;
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) {
auto builtOutputs = ServeProto::Serialise<DrvOutputs>::read(*this, *conn);
for (auto && [output, realisation] : builtOutputs)
status.builtOutputs.insert_or_assign(
std::move(output.outputName),
std::move(realisation));
}
return status;
} }
void buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override void buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override

View file

@ -826,7 +826,7 @@ uint64_t LocalStore::addValidPath(State & state,
state.stmts->RegisterValidPath.use() state.stmts->RegisterValidPath.use()
(printStorePath(info.path)) (printStorePath(info.path))
(info.narHash.to_string(Base16, true)) (info.narHash.to_string(HashFormat::Base16, true))
(info.registrationTime == 0 ? time(0) : info.registrationTime) (info.registrationTime == 0 ? time(0) : info.registrationTime)
(info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver) (info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver)
(info.narSize, info.narSize != 0) (info.narSize, info.narSize != 0)
@ -933,7 +933,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
{ {
state.stmts->UpdatePathInfo.use() state.stmts->UpdatePathInfo.use()
(info.narSize, info.narSize != 0) (info.narSize, info.narSize != 0)
(info.narHash.to_string(Base16, true)) (info.narHash.to_string(HashFormat::Base16, true))
(info.ultimate ? 1 : 0, info.ultimate) (info.ultimate ? 1 : 0, info.ultimate)
(concatStringsSep(" ", info.sigs), !info.sigs.empty()) (concatStringsSep(" ", info.sigs), !info.sigs.empty())
(renderContentAddress(info.ca), (bool) info.ca) (renderContentAddress(info.ca), (bool) info.ca)
@ -1236,7 +1236,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
if (hashResult.first != info.narHash) if (hashResult.first != info.narHash)
throw Error("hash mismatch importing path '%s';\n specified: %s\n got: %s", throw Error("hash mismatch importing path '%s';\n specified: %s\n got: %s",
printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true)); printStorePath(info.path), info.narHash.to_string(HashFormat::Base32, true), hashResult.first.to_string(HashFormat::Base32, true));
if (hashResult.second != info.narSize) if (hashResult.second != info.narSize)
throw Error("size mismatch importing path '%s';\n specified: %s\n got: %s", throw Error("size mismatch importing path '%s';\n specified: %s\n got: %s",
@ -1252,8 +1252,8 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
if (specified.hash != actualHash.hash) { if (specified.hash != actualHash.hash) {
throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s",
printStorePath(info.path), printStorePath(info.path),
specified.hash.to_string(Base32, true), specified.hash.to_string(HashFormat::Base32, true),
actualHash.hash.to_string(Base32, true)); actualHash.hash.to_string(HashFormat::Base32, true));
} }
} }
@ -1545,7 +1545,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
for (auto & link : readDirectory(linksDir)) { for (auto & link : readDirectory(linksDir)) {
printMsg(lvlTalkative, "checking contents of '%s'", link.name); printMsg(lvlTalkative, "checking contents of '%s'", link.name);
Path linkPath = linksDir + "/" + link.name; Path linkPath = linksDir + "/" + link.name;
std::string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false); std::string hash = hashPath(htSHA256, linkPath).first.to_string(HashFormat::Base32, false);
if (hash != link.name) { if (hash != link.name) {
printError("link '%s' was modified! expected hash '%s', got '%s'", printError("link '%s' was modified! expected hash '%s', got '%s'",
linkPath, link.name, hash); linkPath, link.name, hash);
@ -1578,7 +1578,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
if (info->narHash != nullHash && info->narHash != current.first) { if (info->narHash != nullHash && info->narHash != current.first) {
printError("path '%s' was modified! expected hash '%s', got '%s'", printError("path '%s' was modified! expected hash '%s', got '%s'",
printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true)); printStorePath(i), info->narHash.to_string(HashFormat::Base32, true), current.first.to_string(HashFormat::Base32, true));
if (repair) repairPath(i); else errors = true; if (repair) repairPath(i); else errors = true;
} else { } else {

View file

@ -7,6 +7,31 @@
namespace nix { namespace nix {
#if __linux__
static std::vector<gid_t> get_group_list(const char *username, gid_t group_id)
{
std::vector<gid_t> gids;
gids.resize(32); // Initial guess
auto getgroupl_failed {[&] {
int ngroups = gids.size();
int err = getgrouplist(username, group_id, gids.data(), &ngroups);
gids.resize(ngroups);
return err == -1;
}};
// The first error means that the vector was not big enough.
// If it happens again, there is some different problem.
if (getgroupl_failed() && getgroupl_failed()) {
throw SysError("failed to get list of supplementary groups for '%s'", username);
}
return gids;
}
#endif
struct SimpleUserLock : UserLock struct SimpleUserLock : UserLock
{ {
AutoCloseFD fdUserLock; AutoCloseFD fdUserLock;
@ -67,37 +92,14 @@ struct SimpleUserLock : UserLock
throw Error("the Nix user should not be a member of '%s'", settings.buildUsersGroup); throw Error("the Nix user should not be a member of '%s'", settings.buildUsersGroup);
#if __linux__ #if __linux__
/* Get the list of supplementary groups of this build /* Get the list of supplementary groups of this user. This is
user. This is usually either empty or contains a * usually either empty or contains a group such as "kvm". */
group such as "kvm". */
int ngroups = 32; // arbitrary initial guess
std::vector<gid_t> gids;
gids.resize(ngroups);
int err = getgrouplist(
pw->pw_name, pw->pw_gid,
gids.data(),
&ngroups);
/* Our initial size of 32 wasn't sufficient, the
correct size has been stored in ngroups, so we try
again. */
if (err == -1) {
gids.resize(ngroups);
err = getgrouplist(
pw->pw_name, pw->pw_gid,
gids.data(),
&ngroups);
}
// If it failed once more, then something must be broken.
if (err == -1)
throw Error("failed to get list of supplementary groups for '%s'", pw->pw_name);
// Finally, trim back the GID list to its real size. // Finally, trim back the GID list to its real size.
for (auto i = 0; i < ngroups; i++) for (auto gid : get_group_list(pw->pw_name, pw->pw_gid)) {
if (gids[i] != lock->gid) if (gid != lock->gid)
lock->supplementaryGIDs.push_back(gids[i]); lock->supplementaryGIDs.push_back(gid);
}
#endif #endif
return lock; return lock;

View file

@ -332,9 +332,9 @@ public:
(std::string(info->path.name())) (std::string(info->path.name()))
(narInfo ? narInfo->url : "", narInfo != 0) (narInfo ? narInfo->url : "", narInfo != 0)
(narInfo ? narInfo->compression : "", narInfo != 0) (narInfo ? narInfo->compression : "", narInfo != 0)
(narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(Base32, true) : "", narInfo && narInfo->fileHash) (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(HashFormat::Base32, true) : "", narInfo && narInfo->fileHash)
(narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize) (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize)
(info->narHash.to_string(Base32, true)) (info->narHash.to_string(HashFormat::Base32, true))
(info->narSize) (info->narSize)
(concatStringsSep(" ", info->shortRefs())) (concatStringsSep(" ", info->shortRefs()))
(info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver)

View file

@ -105,10 +105,10 @@ std::string NarInfo::to_string(const Store & store) const
assert(compression != ""); assert(compression != "");
res += "Compression: " + compression + "\n"; res += "Compression: " + compression + "\n";
assert(fileHash && fileHash->type == htSHA256); assert(fileHash && fileHash->type == htSHA256);
res += "FileHash: " + fileHash->to_string(Base32, true) + "\n"; res += "FileHash: " + fileHash->to_string(HashFormat::Base32, true) + "\n";
res += "FileSize: " + std::to_string(fileSize) + "\n"; res += "FileSize: " + std::to_string(fileSize) + "\n";
assert(narHash.type == htSHA256); assert(narHash.type == htSHA256);
res += "NarHash: " + narHash.to_string(Base32, true) + "\n"; res += "NarHash: " + narHash.to_string(HashFormat::Base32, true) + "\n";
res += "NarSize: " + std::to_string(narSize) + "\n"; res += "NarSize: " + std::to_string(narSize) + "\n";
res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; res += "References: " + concatStringsSep(" ", shortRefs()) + "\n";

View file

@ -146,10 +146,10 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
contents of the symlink (i.e. the result of readlink()), not contents of the symlink (i.e. the result of readlink()), not
the contents of the target (which may not even exist). */ the contents of the target (which may not even exist). */
Hash hash = hashPath(htSHA256, path).first; Hash hash = hashPath(htSHA256, path).first;
debug("'%1%' has hash '%2%'", path, hash.to_string(Base32, true)); debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Base32, true));
/* Check if this is a known hash. */ /* Check if this is a known hash. */
Path linkPath = linksDir + "/" + hash.to_string(Base32, false); Path linkPath = linksDir + "/" + hash.to_string(HashFormat::Base32, false);
/* Maybe delete the link, if it has been corrupted. */ /* Maybe delete the link, if it has been corrupted. */
if (pathExists(linkPath)) { if (pathExists(linkPath)) {

View file

@ -63,7 +63,7 @@ std::optional<std::pair<std::string_view, ExtendedOutputsSpec>> ExtendedOutputsS
auto specOpt = OutputsSpec::parseOpt(s.substr(found + 1)); auto specOpt = OutputsSpec::parseOpt(s.substr(found + 1));
if (!specOpt) if (!specOpt)
return std::nullopt; return std::nullopt;
return std::pair { s.substr(0, found), ExtendedOutputsSpec::Explicit { *std::move(specOpt) } }; return std::pair { s.substr(0, found), ExtendedOutputsSpec::Explicit { std::move(*specOpt) } };
} }

View file

@ -1,10 +1,27 @@
#include "path-info.hh" #include "path-info.hh"
#include "worker-protocol.hh"
#include "worker-protocol-impl.hh"
#include "store-api.hh" #include "store-api.hh"
namespace nix { namespace nix {
GENERATE_CMP_EXT(
,
UnkeyedValidPathInfo,
me->deriver,
me->narHash,
me->references,
me->registrationTime,
me->narSize,
//me->id,
me->ultimate,
me->sigs,
me->ca);
GENERATE_CMP_EXT(
,
ValidPathInfo,
me->path,
static_cast<const UnkeyedValidPathInfo &>(*me));
std::string ValidPathInfo::fingerprint(const Store & store) const std::string ValidPathInfo::fingerprint(const Store & store) const
{ {
if (narSize == 0) if (narSize == 0)
@ -12,7 +29,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const
store.printStorePath(path)); store.printStorePath(path));
return return
"1;" + store.printStorePath(path) + ";" "1;" + store.printStorePath(path) + ";"
+ narHash.to_string(Base32, true) + ";" + narHash.to_string(HashFormat::Base32, true) + ";"
+ std::to_string(narSize) + ";" + std::to_string(narSize) + ";"
+ concatStringsSep(",", store.printStorePathSet(references)); + concatStringsSep(",", store.printStorePathSet(references));
} }
@ -99,14 +116,13 @@ Strings ValidPathInfo::shortRefs() const
return refs; return refs;
} }
ValidPathInfo::ValidPathInfo( ValidPathInfo::ValidPathInfo(
const Store & store, const Store & store,
std::string_view name, std::string_view name,
ContentAddressWithReferences && ca, ContentAddressWithReferences && ca,
Hash narHash) Hash narHash)
: path(store.makeFixedOutputPathFromCA(name, ca)) : UnkeyedValidPathInfo(narHash)
, narHash(narHash) , path(store.makeFixedOutputPathFromCA(name, ca))
{ {
std::visit(overloaded { std::visit(overloaded {
[this](TextInfo && ti) { [this](TextInfo && ti) {
@ -128,49 +144,4 @@ ValidPathInfo::ValidPathInfo(
}, std::move(ca).raw); }, std::move(ca).raw);
} }
ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format)
{
return read(source, store, format, store.parseStorePath(readString(source)));
}
ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format, StorePath && path)
{
auto deriver = readString(source);
auto narHash = Hash::parseAny(readString(source), htSHA256);
ValidPathInfo info(path, narHash);
if (deriver != "") info.deriver = store.parseStorePath(deriver);
info.references = WorkerProto::Serialise<StorePathSet>::read(store,
WorkerProto::ReadConn { .from = source });
source >> info.registrationTime >> info.narSize;
if (format >= 16) {
source >> info.ultimate;
info.sigs = readStrings<StringSet>(source);
info.ca = ContentAddress::parseOpt(readString(source));
}
return info;
}
void ValidPathInfo::write(
Sink & sink,
const Store & store,
unsigned int format,
bool includePath) const
{
if (includePath)
sink << store.printStorePath(path);
sink << (deriver ? store.printStorePath(*deriver) : "")
<< narHash.to_string(Base16, false);
WorkerProto::write(store,
WorkerProto::WriteConn { .to = sink },
references);
sink << registrationTime << narSize;
if (format >= 16) {
sink << ultimate
<< sigs
<< renderContentAddress(ca);
}
}
} }

View file

@ -29,12 +29,11 @@ struct SubstitutablePathInfo
uint64_t narSize; uint64_t narSize;
}; };
typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos; using SubstitutablePathInfos = std::map<StorePath, SubstitutablePathInfo>;
struct ValidPathInfo struct UnkeyedValidPathInfo
{ {
StorePath path;
std::optional<StorePath> deriver; std::optional<StorePath> deriver;
/** /**
* \todo document this * \todo document this
@ -43,7 +42,7 @@ struct ValidPathInfo
StorePathSet references; StorePathSet references;
time_t registrationTime = 0; time_t registrationTime = 0;
uint64_t narSize = 0; // 0 = unknown uint64_t narSize = 0; // 0 = unknown
uint64_t id; // internal use only uint64_t id = 0; // internal use only
/** /**
* Whether the path is ultimately trusted, that is, it's a * Whether the path is ultimately trusted, that is, it's a
@ -72,13 +71,19 @@ struct ValidPathInfo
*/ */
std::optional<ContentAddress> ca; std::optional<ContentAddress> ca;
bool operator == (const ValidPathInfo & i) const UnkeyedValidPathInfo(const UnkeyedValidPathInfo & other) = default;
{
return UnkeyedValidPathInfo(Hash narHash) : narHash(narHash) { };
path == i.path
&& narHash == i.narHash DECLARE_CMP(UnkeyedValidPathInfo);
&& references == i.references;
} virtual ~UnkeyedValidPathInfo() { }
};
struct ValidPathInfo : UnkeyedValidPathInfo {
StorePath path;
DECLARE_CMP(ValidPathInfo);
/** /**
* Return a fingerprint of the store path to be used in binary * Return a fingerprint of the store path to be used in binary
@ -122,20 +127,15 @@ struct ValidPathInfo
ValidPathInfo(const ValidPathInfo & other) = default; ValidPathInfo(const ValidPathInfo & other) = default;
ValidPathInfo(StorePath && path, Hash narHash) : path(std::move(path)), narHash(narHash) { }; ValidPathInfo(StorePath && path, UnkeyedValidPathInfo info) : UnkeyedValidPathInfo(info), path(std::move(path)) { };
ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { }; ValidPathInfo(const StorePath & path, UnkeyedValidPathInfo info) : UnkeyedValidPathInfo(info), path(path) { };
ValidPathInfo(const Store & store, ValidPathInfo(const Store & store,
std::string_view name, ContentAddressWithReferences && ca, Hash narHash); std::string_view name, ContentAddressWithReferences && ca, Hash narHash);
virtual ~ValidPathInfo() { } virtual ~ValidPathInfo() { }
static ValidPathInfo read(Source & source, const Store & store, unsigned int format);
static ValidPathInfo read(Source & source, const Store & store, unsigned int format, StorePath && path);
void write(Sink & sink, const Store & store, unsigned int format, bool includePath = true) const;
}; };
typedef std::map<StorePath, ValidPathInfo> ValidPathInfos; using ValidPathInfos = std::map<StorePath, ValidPathInfo>;
} }

View file

@ -35,7 +35,7 @@ StorePath::StorePath(std::string_view _baseName)
} }
StorePath::StorePath(const Hash & hash, std::string_view _name) StorePath::StorePath(const Hash & hash, std::string_view _name)
: baseName((hash.to_string(Base32, false) + "-").append(std::string(_name))) : baseName((hash.to_string(HashFormat::Base32, false) + "-").append(std::string(_name)))
{ {
checkName(baseName, name()); checkName(baseName, name());
} }

View file

@ -39,7 +39,7 @@ struct DrvOutput {
std::string to_string() const; std::string to_string() const;
std::string strHash() const std::string strHash() const
{ return drvHash.to_string(Base16, true); } { return drvHash.to_string(HashFormat::Base16, true); }
static DrvOutput parse(const std::string &); static DrvOutput parse(const std::string &);

View file

@ -30,7 +30,7 @@ struct RemoteStore::Connection
* sides support. (If the maximum doesn't exist, we would fail to * sides support. (If the maximum doesn't exist, we would fail to
* establish a connection and produce a value of this type.) * establish a connection and produce a value of this type.)
*/ */
unsigned int daemonVersion; WorkerProto::Version daemonVersion;
/** /**
* Whether the remote side trusts us or not. * Whether the remote side trusts us or not.
@ -70,6 +70,7 @@ struct RemoteStore::Connection
{ {
return WorkerProto::ReadConn { return WorkerProto::ReadConn {
.from = from, .from = from,
.version = daemonVersion,
}; };
} }
@ -85,6 +86,7 @@ struct RemoteStore::Connection
{ {
return WorkerProto::WriteConn { return WorkerProto::WriteConn {
.to = to, .to = to,
.version = daemonVersion,
}; };
} }

View file

@ -332,7 +332,8 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path,
if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path)); if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path));
} }
info = std::make_shared<ValidPathInfo>( info = std::make_shared<ValidPathInfo>(
ValidPathInfo::read(conn->from, *this, GET_PROTOCOL_MINOR(conn->daemonVersion), StorePath{path})); StorePath{path},
WorkerProto::Serialise<UnkeyedValidPathInfo>::read(*this, *conn));
} }
callback(std::move(info)); callback(std::move(info));
} catch (...) { callback.rethrow(); } } catch (...) { callback.rethrow(); }
@ -445,7 +446,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
} }
return make_ref<ValidPathInfo>( return make_ref<ValidPathInfo>(
ValidPathInfo::read(conn->from, *this, GET_PROTOCOL_MINOR(conn->daemonVersion))); WorkerProto::Serialise<ValidPathInfo>::read(*this, *conn));
} }
else { else {
if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25"); if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25");
@ -541,7 +542,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
conn->to << WorkerProto::Op::AddToStoreNar conn->to << WorkerProto::Op::AddToStoreNar
<< printStorePath(info.path) << printStorePath(info.path)
<< (info.deriver ? printStorePath(*info.deriver) : "") << (info.deriver ? printStorePath(*info.deriver) : "")
<< info.narHash.to_string(Base16, false); << info.narHash.to_string(HashFormat::Base16, false);
WorkerProto::write(*this, *conn, info.references); WorkerProto::write(*this, *conn, info.references);
conn->to << info.registrationTime << info.narSize conn->to << info.registrationTime << info.narSize
<< info.ultimate << info.sigs << renderContentAddress(info.ca) << info.ultimate << info.sigs << renderContentAddress(info.ca)
@ -570,7 +571,12 @@ void RemoteStore::addMultipleToStore(
auto source = sinkToSource([&](Sink & sink) { auto source = sinkToSource([&](Sink & sink) {
sink << pathsToCopy.size(); sink << pathsToCopy.size();
for (auto & [pathInfo, pathSource] : pathsToCopy) { for (auto & [pathInfo, pathSource] : pathsToCopy) {
pathInfo.write(sink, *this, 16); WorkerProto::Serialise<ValidPathInfo>::write(*this,
WorkerProto::WriteConn {
.to = sink,
.version = 16,
},
pathInfo);
pathSource->drainInto(sink); pathSource->drainInto(sink);
} }
}); });
@ -655,33 +661,6 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id,
} catch (...) { return callback.rethrow(); } } catch (...) { return callback.rethrow(); }
} }
static void writeDerivedPaths(RemoteStore & store, RemoteStore::Connection & conn, const std::vector<DerivedPath> & reqs)
{
if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 30) {
WorkerProto::write(store, conn, reqs);
} else {
Strings ss;
for (auto & p : reqs) {
auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p);
std::visit(overloaded {
[&](const StorePathWithOutputs & s) {
ss.push_back(s.to_string(store));
},
[&](const StorePath & drvPath) {
throw Error("trying to request '%s', but daemon protocol %d.%d is too old (< 1.29) to request a derivation file",
store.printStorePath(drvPath),
GET_PROTOCOL_MAJOR(conn.daemonVersion),
GET_PROTOCOL_MINOR(conn.daemonVersion));
},
[&](std::monostate) {
throw Error("wanted to build a derivation that is itself a build product, but the legacy 'ssh://' protocol doesn't support that. Try using 'ssh-ng://'");
},
}, sOrDrvPath);
}
conn.to << ss;
}
}
void RemoteStore::copyDrvsFromEvalStore( void RemoteStore::copyDrvsFromEvalStore(
const std::vector<DerivedPath> & paths, const std::vector<DerivedPath> & paths,
std::shared_ptr<Store> evalStore) std::shared_ptr<Store> evalStore)
@ -711,7 +690,7 @@ void RemoteStore::buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMod
auto conn(getConnection()); auto conn(getConnection());
conn->to << WorkerProto::Op::BuildPaths; conn->to << WorkerProto::Op::BuildPaths;
assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13); assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13);
writeDerivedPaths(*this, *conn, drvPaths); WorkerProto::write(*this, *conn, drvPaths);
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15)
conn->to << buildMode; conn->to << buildMode;
else else
@ -735,7 +714,7 @@ std::vector<KeyedBuildResult> RemoteStore::buildPathsWithResults(
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 34) { if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 34) {
conn->to << WorkerProto::Op::BuildPathsWithResults; conn->to << WorkerProto::Op::BuildPathsWithResults;
writeDerivedPaths(*this, *conn, paths); WorkerProto::write(*this, *conn, paths);
conn->to << buildMode; conn->to << buildMode;
conn.processStderr(); conn.processStderr();
return WorkerProto::Serialise<std::vector<KeyedBuildResult>>::read(*this, *conn); return WorkerProto::Serialise<std::vector<KeyedBuildResult>>::read(*this, *conn);
@ -815,20 +794,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD
writeDerivation(conn->to, *this, drv); writeDerivation(conn->to, *this, drv);
conn->to << buildMode; conn->to << buildMode;
conn.processStderr(); conn.processStderr();
BuildResult res; return WorkerProto::Serialise<BuildResult>::read(*this, *conn);
res.status = (BuildResult::Status) readInt(conn->from);
conn->from >> res.errorMsg;
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) {
conn->from >> res.timesBuilt >> res.isNonDeterministic >> res.startTime >> res.stopTime;
}
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 28) {
auto builtOutputs = WorkerProto::Serialise<DrvOutputs>::read(*this, *conn);
for (auto && [output, realisation] : builtOutputs)
res.builtOutputs.insert_or_assign(
std::move(output.outputName),
std::move(realisation));
}
return res;
} }
@ -929,7 +895,7 @@ void RemoteStore::queryMissing(const std::vector<DerivedPath> & targets,
// to prevent a deadlock. // to prevent a deadlock.
goto fallback; goto fallback;
conn->to << WorkerProto::Op::QueryMissing; conn->to << WorkerProto::Op::QueryMissing;
writeDerivedPaths(*this, *conn, targets); WorkerProto::write(*this, *conn, targets);
conn.processStderr(); conn.processStderr();
willBuild = WorkerProto::Serialise<StorePathSet>::read(*this, *conn); willBuild = WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
willSubstitute = WorkerProto::Serialise<StorePathSet>::read(*this, *conn); willSubstitute = WorkerProto::Serialise<StorePathSet>::read(*this, *conn);

View file

@ -2,7 +2,103 @@ R"(
**Store URL format**: `s3://`*bucket-name* **Store URL format**: `s3://`*bucket-name*
This store allows reading and writing a binary cache stored in an AWS This store allows reading and writing a binary cache stored in an AWS S3 (or S3-compatible service) bucket.
S3 bucket. This store shares many idioms with the [HTTP Binary Cache Store](#http-binary-cache-store).
For AWS S3, the binary cache URL for a bucket named `example-nix-cache` will be exactly <s3://example-nix-cache>.
For S3 compatible binary caches, consult that cache's documentation.
### Anonymous reads to your S3-compatible binary cache
> If your binary cache is publicly accessible and does not require authentication,
> it is simplest to use the [HTTP Binary Cache Store] rather than S3 Binary Cache Store with
> <https://example-nix-cache.s3.amazonaws.com> instead of <s3://example-nix-cache>.
Your bucket will need a
[bucket policy](https://docs.aws.amazon.com/AmazonS3/v1/userguide/bucket-policies.html)
like the following to be accessible:
```json
{
"Id": "DirectReads",
"Version": "2012-10-17",
"Statement": [
{
"Sid": "AllowDirectReads",
"Action": [
"s3:GetObject",
"s3:GetBucketLocation"
],
"Effect": "Allow",
"Resource": [
"arn:aws:s3:::example-nix-cache",
"arn:aws:s3:::example-nix-cache/*"
],
"Principal": "*"
}
]
}
```
### Authentication
Nix will use the
[default credential provider chain](https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html)
for authenticating requests to Amazon S3.
Note that this means Nix will read environment variables and files with different idioms than with Nix's own settings, as implemented by the AWS SDK.
Consult the documentation linked above for further details.
### Authenticated reads to your S3 binary cache
Your bucket will need a bucket policy allowing the desired users to perform the `s3:GetObject` and `s3:GetBucketLocation` action on all objects in the bucket.
The [anonymous policy given above](#anonymous-reads-to-your-s3-compatible-binary-cache) can be updated to have a restricted `Principal` to support this.
### Authenticated writes to your S3-compatible binary cache
Your account will need an IAM policy to support uploading to the bucket:
```json
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "UploadToCache",
"Effect": "Allow",
"Action": [
"s3:AbortMultipartUpload",
"s3:GetBucketLocation",
"s3:GetObject",
"s3:ListBucket",
"s3:ListBucketMultipartUploads",
"s3:ListMultipartUploadParts",
"s3:PutObject"
],
"Resource": [
"arn:aws:s3:::example-nix-cache",
"arn:aws:s3:::example-nix-cache/*"
]
}
]
}
```
### Examples
With bucket policies and authentication set up as described above, uploading works via [`nix copy`](@docroot@/command-ref/new-cli/nix3-copy.md) (experimental).
- To upload with a specific credential profile for Amazon S3:
```console
$ nix copy nixpkgs.hello \
--to 's3://example-nix-cache?profile=cache-upload&region=eu-west-2'
```
- To upload to an S3-compatible binary cache:
```console
$ nix copy nixpkgs.hello --to \
's3://example-nix-cache?profile=cache-upload&scheme=https&endpoint=minio.example.com'
```
)" )"

View file

@ -2,6 +2,7 @@
#include "util.hh" #include "util.hh"
#include "path-with-outputs.hh" #include "path-with-outputs.hh"
#include "store-api.hh" #include "store-api.hh"
#include "build-result.hh"
#include "serve-protocol.hh" #include "serve-protocol.hh"
#include "serve-protocol-impl.hh" #include "serve-protocol-impl.hh"
#include "archive.hh" #include "archive.hh"
@ -12,4 +13,46 @@ namespace nix {
/* protocol-specific definitions */ /* protocol-specific definitions */
BuildResult ServeProto::Serialise<BuildResult>::read(const Store & store, ServeProto::ReadConn conn)
{
BuildResult status;
status.status = (BuildResult::Status) readInt(conn.from);
conn.from >> status.errorMsg;
if (GET_PROTOCOL_MINOR(conn.version) >= 3)
conn.from
>> status.timesBuilt
>> status.isNonDeterministic
>> status.startTime
>> status.stopTime;
if (GET_PROTOCOL_MINOR(conn.version) >= 6) {
auto builtOutputs = ServeProto::Serialise<DrvOutputs>::read(store, conn);
for (auto && [output, realisation] : builtOutputs)
status.builtOutputs.insert_or_assign(
std::move(output.outputName),
std::move(realisation));
}
return status;
}
void ServeProto::Serialise<BuildResult>::write(const Store & store, ServeProto::WriteConn conn, const BuildResult & status)
{
conn.to
<< status.status
<< status.errorMsg;
if (GET_PROTOCOL_MINOR(conn.version) >= 3)
conn.to
<< status.timesBuilt
<< status.isNonDeterministic
<< status.startTime
<< status.stopTime;
if (GET_PROTOCOL_MINOR(conn.version) >= 6) {
DrvOutputs builtOutputs;
for (auto & [output, realisation] : status.builtOutputs)
builtOutputs.insert_or_assign(realisation.id, realisation);
ServeProto::write(store, conn, builtOutputs);
}
}
} }

View file

@ -16,6 +16,9 @@ namespace nix {
class Store; class Store;
struct Source; struct Source;
// items being serialised
struct BuildResult;
/** /**
* The "serve protocol", used by ssh:// stores. * The "serve protocol", used by ssh:// stores.
@ -30,26 +33,29 @@ struct ServeProto
*/ */
enum struct Command : uint64_t; enum struct Command : uint64_t;
/**
* Version type for the protocol.
*
* @todo Convert to struct with separate major vs minor fields.
*/
using Version = unsigned int;
/** /**
* A unidirectional read connection, to be used by the read half of the * A unidirectional read connection, to be used by the read half of the
* canonical serializers below. * canonical serializers below.
*
* This currently is just a `Source &`, but more fields will be added
* later.
*/ */
struct ReadConn { struct ReadConn {
Source & from; Source & from;
Version version;
}; };
/** /**
* A unidirectional write connection, to be used by the write half of the * A unidirectional write connection, to be used by the write half of the
* canonical serializers below. * canonical serializers below.
*
* This currently is just a `Sink &`, but more fields will be added
* later.
*/ */
struct WriteConn { struct WriteConn {
Sink & to; Sink & to;
Version version;
}; };
/** /**
@ -133,6 +139,9 @@ inline std::ostream & operator << (std::ostream & s, ServeProto::Command op)
static void write(const Store & store, ServeProto::WriteConn conn, const T & t); \ static void write(const Store & store, ServeProto::WriteConn conn, const T & t); \
}; };
template<>
DECLARE_SERVE_SERIALISER(BuildResult);
template<typename T> template<typename T>
DECLARE_SERVE_SERIALISER(std::vector<T>); DECLARE_SERVE_SERIALISER(std::vector<T>);
template<typename T> template<typename T>

View file

@ -11,6 +11,9 @@
#include "archive.hh" #include "archive.hh"
#include "callback.hh" #include "callback.hh"
#include "remote-store.hh" #include "remote-store.hh"
// FIXME this should not be here, see TODO below on
// `addMultipleToStore`.
#include "worker-protocol.hh"
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
#include <regex> #include <regex>
@ -154,7 +157,7 @@ StorePath Store::makeStorePath(std::string_view type,
StorePath Store::makeStorePath(std::string_view type, StorePath Store::makeStorePath(std::string_view type,
const Hash & hash, std::string_view name) const const Hash & hash, std::string_view name) const
{ {
return makeStorePath(type, hash.to_string(Base16, true), name); return makeStorePath(type, hash.to_string(HashFormat::Base16, true), name);
} }
@ -192,7 +195,7 @@ StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInf
hashString(htSHA256, hashString(htSHA256,
"fixed:out:" "fixed:out:"
+ makeFileIngestionPrefix(info.method) + makeFileIngestionPrefix(info.method)
+ info.hash.to_string(Base16, true) + ":"), + info.hash.to_string(HashFormat::Base16, true) + ":"),
name); name);
} }
} }
@ -225,12 +228,16 @@ StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentA
} }
std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name, std::pair<StorePath, Hash> Store::computeStorePathFromDump(
const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, PathFilter & filter) const Source & dump,
std::string_view name,
FileIngestionMethod method,
HashType hashAlgo,
const StorePathSet & references) const
{ {
Hash h = method == FileIngestionMethod::Recursive HashSink sink(hashAlgo);
? hashPath(hashAlgo, srcPath, filter).first dump.drainInto(sink);
: hashFile(hashAlgo, srcPath); auto h = sink.finish().first;
FixedOutputInfo caInfo { FixedOutputInfo caInfo {
.method = method, .method = method,
.hash = h, .hash = h,
@ -357,7 +364,13 @@ void Store::addMultipleToStore(
{ {
auto expected = readNum<uint64_t>(source); auto expected = readNum<uint64_t>(source);
for (uint64_t i = 0; i < expected; ++i) { for (uint64_t i = 0; i < expected; ++i) {
auto info = ValidPathInfo::read(source, *this, 16); // FIXME we should not be using the worker protocol here, let
// alone the worker protocol with a hard-coded version!
auto info = WorkerProto::Serialise<ValidPathInfo>::read(*this,
WorkerProto::ReadConn {
.from = source,
.version = 16,
});
info.ultimate = false; info.ultimate = false;
addToStore(info, source, repair, checkSigs); addToStore(info, source, repair, checkSigs);
} }
@ -884,7 +897,7 @@ std::string Store::makeValidityRegistration(const StorePathSet & paths,
auto info = queryPathInfo(i); auto info = queryPathInfo(i);
if (showHash) { if (showHash) {
s += info->narHash.to_string(Base16, false) + "\n"; s += info->narHash.to_string(HashFormat::Base16, false) + "\n";
s += fmt("%1%\n", info->narSize); s += fmt("%1%\n", info->narSize);
} }
@ -938,7 +951,7 @@ StorePathSet Store::exportReferences(const StorePathSet & storePaths, const Stor
json Store::pathInfoToJSON(const StorePathSet & storePaths, json Store::pathInfoToJSON(const StorePathSet & storePaths,
bool includeImpureInfo, bool showClosureSize, bool includeImpureInfo, bool showClosureSize,
Base hashBase, HashFormat hashFormat,
AllowInvalidFlag allowInvalid) AllowInvalidFlag allowInvalid)
{ {
json::array_t jsonList = json::array(); json::array_t jsonList = json::array();
@ -951,7 +964,7 @@ json Store::pathInfoToJSON(const StorePathSet & storePaths,
jsonPath["path"] = printStorePath(info->path); jsonPath["path"] = printStorePath(info->path);
jsonPath["valid"] = true; jsonPath["valid"] = true;
jsonPath["narHash"] = info->narHash.to_string(hashBase, true); jsonPath["narHash"] = info->narHash.to_string(hashFormat, true);
jsonPath["narSize"] = info->narSize; jsonPath["narSize"] = info->narSize;
{ {
@ -993,7 +1006,7 @@ json Store::pathInfoToJSON(const StorePathSet & storePaths,
if (!narInfo->url.empty()) if (!narInfo->url.empty())
jsonPath["url"] = narInfo->url; jsonPath["url"] = narInfo->url;
if (narInfo->fileHash) if (narInfo->fileHash)
jsonPath["downloadHash"] = narInfo->fileHash->to_string(hashBase, true); jsonPath["downloadHash"] = narInfo->fileHash->to_string(hashFormat, true);
if (narInfo->fileSize) if (narInfo->fileSize)
jsonPath["downloadSize"] = narInfo->fileSize; jsonPath["downloadSize"] = narInfo->fileSize;
if (showClosureSize) if (showClosureSize)

View file

@ -292,14 +292,15 @@ public:
StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const; StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const;
/** /**
* Preparatory part of addToStore(). * Read-only variant of addToStoreFromDump(). It returns the store
* * path to which a NAR or flat file would be written.
* @return the store path to which srcPath is to be copied
* and the cryptographic hash of the contents of srcPath.
*/ */
std::pair<StorePath, Hash> computeStorePathForPath(std::string_view name, std::pair<StorePath, Hash> computeStorePathFromDump(
const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive, Source & dump,
HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const; std::string_view name,
FileIngestionMethod method = FileIngestionMethod::Recursive,
HashType hashAlgo = htSHA256,
const StorePathSet & references = {}) const;
/** /**
* Preparatory part of addTextToStore(). * Preparatory part of addTextToStore().
@ -676,7 +677,7 @@ public:
*/ */
nlohmann::json pathInfoToJSON(const StorePathSet & storePaths, nlohmann::json pathInfoToJSON(const StorePathSet & storePaths,
bool includeImpureInfo, bool showClosureSize, bool includeImpureInfo, bool showClosureSize,
Base hashBase = Base32, HashFormat hashFormat = HashFormat::Base32,
AllowInvalidFlag allowInvalid = DisallowInvalid); AllowInvalidFlag allowInvalid = DisallowInvalid);
/** /**

View file

@ -20,4 +20,9 @@ static bool testAccept() {
return getEnv("_NIX_TEST_ACCEPT") == "1"; return getEnv("_NIX_TEST_ACCEPT") == "1";
} }
constexpr std::string_view cannotReadGoldenMaster =
"Cannot read golden master because another test is also updating it";
constexpr std::string_view updatingGoldenMaster =
"Updating golden master";
} }

View file

@ -13,10 +13,71 @@ namespace nix {
const char commonProtoDir[] = "common-protocol"; const char commonProtoDir[] = "common-protocol";
using CommonProtoTest = ProtoTest<CommonProto, commonProtoDir>; class CommonProtoTest : public ProtoTest<CommonProto, commonProtoDir>
{
public:
/**
* Golden test for `T` reading
*/
template<typename T>
void readTest(PathView testStem, T value)
{
if (testAccept())
{
GTEST_SKIP() << cannotReadGoldenMaster;
}
else
{
auto encoded = readFile(goldenMaster(testStem));
T got = ({
StringSource from { encoded };
CommonProto::Serialise<T>::read(
*store,
CommonProto::ReadConn { .from = from });
});
ASSERT_EQ(got, value);
}
}
/**
* Golden test for `T` write
*/
template<typename T>
void writeTest(PathView testStem, const T & value)
{
auto file = goldenMaster(testStem);
StringSink to;
CommonProto::write(
*store,
CommonProto::WriteConn { .to = to },
value);
if (testAccept())
{
createDirs(dirOf(file));
writeFile(file, to.s);
GTEST_SKIP() << updatingGoldenMaster;
}
else
{
auto expected = readFile(file);
ASSERT_EQ(to.s, expected);
}
}
};
#define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \
TEST_F(CommonProtoTest, NAME ## _read) { \
readTest(STEM, VALUE); \
} \
TEST_F(CommonProtoTest, NAME ## _write) { \
writeTest(STEM, VALUE); \
}
CHARACTERIZATION_TEST( CHARACTERIZATION_TEST(
CommonProtoTest,
string, string,
"string", "string",
(std::tuple<std::string, std::string, std::string, std::string, std::string> { (std::tuple<std::string, std::string, std::string, std::string, std::string> {
@ -28,7 +89,6 @@ CHARACTERIZATION_TEST(
})) }))
CHARACTERIZATION_TEST( CHARACTERIZATION_TEST(
CommonProtoTest,
storePath, storePath,
"store-path", "store-path",
(std::tuple<StorePath, StorePath> { (std::tuple<StorePath, StorePath> {
@ -37,7 +97,6 @@ CHARACTERIZATION_TEST(
})) }))
CHARACTERIZATION_TEST( CHARACTERIZATION_TEST(
CommonProtoTest,
contentAddress, contentAddress,
"content-address", "content-address",
(std::tuple<ContentAddress, ContentAddress, ContentAddress> { (std::tuple<ContentAddress, ContentAddress, ContentAddress> {
@ -56,7 +115,6 @@ CHARACTERIZATION_TEST(
})) }))
CHARACTERIZATION_TEST( CHARACTERIZATION_TEST(
CommonProtoTest,
drvOutput, drvOutput,
"drv-output", "drv-output",
(std::tuple<DrvOutput, DrvOutput> { (std::tuple<DrvOutput, DrvOutput> {
@ -71,7 +129,6 @@ CHARACTERIZATION_TEST(
})) }))
CHARACTERIZATION_TEST( CHARACTERIZATION_TEST(
CommonProtoTest,
realisation, realisation,
"realisation", "realisation",
(std::tuple<Realisation, Realisation> { (std::tuple<Realisation, Realisation> {
@ -103,7 +160,6 @@ CHARACTERIZATION_TEST(
})) }))
CHARACTERIZATION_TEST( CHARACTERIZATION_TEST(
CommonProtoTest,
vector, vector,
"vector", "vector",
(std::tuple<std::vector<std::string>, std::vector<std::string>, std::vector<std::string>, std::vector<std::vector<std::string>>> { (std::tuple<std::vector<std::string>, std::vector<std::string>, std::vector<std::string>, std::vector<std::vector<std::string>>> {
@ -114,7 +170,6 @@ CHARACTERIZATION_TEST(
})) }))
CHARACTERIZATION_TEST( CHARACTERIZATION_TEST(
CommonProtoTest,
set, set,
"set", "set",
(std::tuple<std::set<std::string>, std::set<std::string>, std::set<std::string>, std::set<std::set<std::string>>> { (std::tuple<std::set<std::string>, std::set<std::string>, std::set<std::string>, std::set<std::set<std::string>>> {
@ -125,7 +180,6 @@ CHARACTERIZATION_TEST(
})) }))
CHARACTERIZATION_TEST( CHARACTERIZATION_TEST(
CommonProtoTest,
optionalStorePath, optionalStorePath,
"optional-store-path", "optional-store-path",
(std::tuple<std::optional<StorePath>, std::optional<StorePath>> { (std::tuple<std::optional<StorePath>, std::optional<StorePath>> {
@ -136,7 +190,6 @@ CHARACTERIZATION_TEST(
})) }))
CHARACTERIZATION_TEST( CHARACTERIZATION_TEST(
CommonProtoTest,
optionalContentAddress, optionalContentAddress,
"optional-content-address", "optional-content-address",
(std::tuple<std::optional<ContentAddress>, std::optional<ContentAddress>> { (std::tuple<std::optional<ContentAddress>, std::optional<ContentAddress>> {

View file

@ -5,9 +5,12 @@
#include "derivations.hh" #include "derivations.hh"
#include "tests/libstore.hh" #include "tests/libstore.hh"
#include "tests/characterization.hh"
namespace nix { namespace nix {
using nlohmann::json;
class DerivationTest : public LibStoreTest class DerivationTest : public LibStoreTest
{ {
public: public:
@ -16,6 +19,12 @@ public:
* to worry about race conditions if the tests run concurrently. * to worry about race conditions if the tests run concurrently.
*/ */
ExperimentalFeatureSettings mockXpSettings; ExperimentalFeatureSettings mockXpSettings;
Path unitTestData = getUnitTestData() + "/libstore/derivation";
Path goldenMaster(std::string_view testStem) {
return unitTestData + "/" + testStem;
}
}; };
class CaDerivationTest : public DerivationTest class CaDerivationTest : public DerivationTest
@ -46,7 +55,7 @@ TEST_F(DerivationTest, BadATerm_version) {
ASSERT_THROW( ASSERT_THROW(
parseDerivation( parseDerivation(
*store, *store,
R"(DrvWithVersion("invalid-version",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))", readFile(goldenMaster("bad-version.drv")),
"whatever", "whatever",
mockXpSettings), mockXpSettings),
FormatError); FormatError);
@ -56,50 +65,61 @@ TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps) {
ASSERT_THROW( ASSERT_THROW(
parseDerivation( parseDerivation(
*store, *store,
R"(Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))", readFile(goldenMaster("bad-old-version-dyn-deps.drv")),
"dyn-dep-derivation", "dyn-dep-derivation",
mockXpSettings), mockXpSettings),
FormatError); FormatError);
} }
#define TEST_JSON(FIXTURE, NAME, STR, VAL, DRV_NAME, OUTPUT_NAME) \ #define TEST_JSON(FIXTURE, NAME, VAL, DRV_NAME, OUTPUT_NAME) \
TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _to_json) { \
using nlohmann::literals::operator "" _json; \
ASSERT_EQ( \
STR ## _json, \
(DerivationOutput { VAL }).toJSON( \
*store, \
DRV_NAME, \
OUTPUT_NAME)); \
} \
\
TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _from_json) { \ TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _from_json) { \
using nlohmann::literals::operator "" _json; \ if (testAccept()) \
ASSERT_EQ( \ { \
DerivationOutput { VAL }, \ GTEST_SKIP() << cannotReadGoldenMaster; \
DerivationOutput::fromJSON( \ } \
else \
{ \
auto encoded = json::parse( \
readFile(goldenMaster("output-" #NAME ".json"))); \
DerivationOutput got = DerivationOutput::fromJSON( \
*store, \ *store, \
DRV_NAME, \ DRV_NAME, \
OUTPUT_NAME, \ OUTPUT_NAME, \
STR ## _json, \ encoded, \
mockXpSettings)); \ mockXpSettings); \
DerivationOutput expected { VAL }; \
ASSERT_EQ(got, expected); \
} \
} \
\
TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _to_json) { \
auto file = goldenMaster("output-" #NAME ".json"); \
\
json got = DerivationOutput { VAL }.toJSON( \
*store, \
DRV_NAME, \
OUTPUT_NAME); \
\
if (testAccept()) \
{ \
createDirs(dirOf(file)); \
writeFile(file, got.dump(2) + "\n"); \
GTEST_SKIP() << updatingGoldenMaster; \
} \
else \
{ \
auto expected = json::parse(readFile(file)); \
ASSERT_EQ(got, expected); \
} \
} }
TEST_JSON(DerivationTest, inputAddressed, TEST_JSON(DerivationTest, inputAddressed,
R"({
"path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"
})",
(DerivationOutput::InputAddressed { (DerivationOutput::InputAddressed {
.path = store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"), .path = store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"),
}), }),
"drv-name", "output-name") "drv-name", "output-name")
TEST_JSON(DerivationTest, caFixedFlat, TEST_JSON(DerivationTest, caFixedFlat,
R"({
"hashAlgo": "sha256",
"hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
"path": "/nix/store/rhcg9h16sqvlbpsa6dqm57sbr2al6nzg-drv-name-output-name"
})",
(DerivationOutput::CAFixed { (DerivationOutput::CAFixed {
.ca = { .ca = {
.method = FileIngestionMethod::Flat, .method = FileIngestionMethod::Flat,
@ -109,11 +129,6 @@ TEST_JSON(DerivationTest, caFixedFlat,
"drv-name", "output-name") "drv-name", "output-name")
TEST_JSON(DerivationTest, caFixedNAR, TEST_JSON(DerivationTest, caFixedNAR,
R"({
"hashAlgo": "r:sha256",
"hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
"path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"
})",
(DerivationOutput::CAFixed { (DerivationOutput::CAFixed {
.ca = { .ca = {
.method = FileIngestionMethod::Recursive, .method = FileIngestionMethod::Recursive,
@ -123,11 +138,6 @@ TEST_JSON(DerivationTest, caFixedNAR,
"drv-name", "output-name") "drv-name", "output-name")
TEST_JSON(DynDerivationTest, caFixedText, TEST_JSON(DynDerivationTest, caFixedText,
R"({
"hashAlgo": "text:sha256",
"hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
"path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name"
})",
(DerivationOutput::CAFixed { (DerivationOutput::CAFixed {
.ca = { .ca = {
.hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="),
@ -136,9 +146,6 @@ TEST_JSON(DynDerivationTest, caFixedText,
"drv-name", "output-name") "drv-name", "output-name")
TEST_JSON(CaDerivationTest, caFloating, TEST_JSON(CaDerivationTest, caFloating,
R"({
"hashAlgo": "r:sha256"
})",
(DerivationOutput::CAFloating { (DerivationOutput::CAFloating {
.method = FileIngestionMethod::Recursive, .method = FileIngestionMethod::Recursive,
.hashType = htSHA256, .hashType = htSHA256,
@ -146,15 +153,10 @@ TEST_JSON(CaDerivationTest, caFloating,
"drv-name", "output-name") "drv-name", "output-name")
TEST_JSON(DerivationTest, deferred, TEST_JSON(DerivationTest, deferred,
R"({ })",
DerivationOutput::Deferred { }, DerivationOutput::Deferred { },
"drv-name", "output-name") "drv-name", "output-name")
TEST_JSON(ImpureDerivationTest, impure, TEST_JSON(ImpureDerivationTest, impure,
R"({
"hashAlgo": "r:sha256",
"impure": true
})",
(DerivationOutput::Impure { (DerivationOutput::Impure {
.method = FileIngestionMethod::Recursive, .method = FileIngestionMethod::Recursive,
.hashType = htSHA256, .hashType = htSHA256,
@ -163,43 +165,79 @@ TEST_JSON(ImpureDerivationTest, impure,
#undef TEST_JSON #undef TEST_JSON
#define TEST_JSON(FIXTURE, NAME, STR, VAL) \ #define TEST_JSON(FIXTURE, NAME, VAL) \
TEST_F(FIXTURE, Derivation_ ## NAME ## _to_json) { \ TEST_F(FIXTURE, Derivation_ ## NAME ## _from_json) { \
using nlohmann::literals::operator "" _json; \ if (testAccept()) \
ASSERT_EQ( \ { \
STR ## _json, \ GTEST_SKIP() << cannotReadGoldenMaster; \
(VAL).toJSON(*store)); \ } \
else \
{ \
auto encoded = json::parse( \
readFile(goldenMaster( #NAME ".json"))); \
Derivation expected { VAL }; \
Derivation got = Derivation::fromJSON( \
*store, \
encoded, \
mockXpSettings); \
ASSERT_EQ(got, expected); \
} \
} \ } \
\ \
TEST_F(FIXTURE, Derivation_ ## NAME ## _from_json) { \ TEST_F(FIXTURE, Derivation_ ## NAME ## _to_json) { \
using nlohmann::literals::operator "" _json; \ auto file = goldenMaster( #NAME ".json"); \
ASSERT_EQ( \ \
(VAL), \ json got = Derivation { VAL }.toJSON(*store); \
Derivation::fromJSON( \ \
*store, \ if (testAccept()) \
STR ## _json, \ { \
mockXpSettings)); \ createDirs(dirOf(file)); \
writeFile(file, got.dump(2) + "\n"); \
GTEST_SKIP() << updatingGoldenMaster; \
} \
else \
{ \
auto expected = json::parse(readFile(file)); \
ASSERT_EQ(got, expected); \
} \
} }
#define TEST_ATERM(FIXTURE, NAME, STR, VAL, DRV_NAME) \ #define TEST_ATERM(FIXTURE, NAME, VAL, DRV_NAME) \
TEST_F(FIXTURE, Derivation_ ## NAME ## _to_aterm) { \
ASSERT_EQ( \
STR, \
(VAL).unparse(*store, false)); \
} \
\
TEST_F(FIXTURE, Derivation_ ## NAME ## _from_aterm) { \ TEST_F(FIXTURE, Derivation_ ## NAME ## _from_aterm) { \
auto parsed = parseDerivation( \ if (testAccept()) \
{ \
GTEST_SKIP() << cannotReadGoldenMaster; \
} \
else \
{ \
auto encoded = readFile(goldenMaster( #NAME ".drv")); \
Derivation expected { VAL }; \
auto got = parseDerivation( \
*store, \ *store, \
STR, \ std::move(encoded), \
DRV_NAME, \ DRV_NAME, \
mockXpSettings); \ mockXpSettings); \
ASSERT_EQ( \ ASSERT_EQ(got.toJSON(*store), expected.toJSON(*store)) ; \
(VAL).toJSON(*store), \ ASSERT_EQ(got, expected); \
parsed.toJSON(*store)); \ } \
ASSERT_EQ( \ } \
(VAL), \ \
parsed); \ TEST_F(FIXTURE, Derivation_ ## NAME ## _to_aterm) { \
auto file = goldenMaster( #NAME ".drv"); \
\
auto got = (VAL).unparse(*store, false); \
\
if (testAccept()) \
{ \
createDirs(dirOf(file)); \
writeFile(file, got); \
GTEST_SKIP() << updatingGoldenMaster; \
} \
else \
{ \
auto expected = readFile(file); \
ASSERT_EQ(got, expected); \
} \
} }
Derivation makeSimpleDrv(const Store & store) { Derivation makeSimpleDrv(const Store & store) {
@ -236,36 +274,9 @@ Derivation makeSimpleDrv(const Store & store) {
return drv; return drv;
} }
TEST_JSON(DerivationTest, simple, TEST_JSON(DerivationTest, simple, makeSimpleDrv(*store))
R"({
"name": "simple-derivation",
"inputSrcs": [
"/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"
],
"inputDrvs": {
"/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": {
"dynamicOutputs": {},
"outputs": [
"cat",
"dog"
]
}
},
"system": "wasm-sel4",
"builder": "foo",
"args": [
"bar",
"baz"
],
"env": {
"BIG_BAD": "WOLF"
},
"outputs": {}
})",
makeSimpleDrv(*store))
TEST_ATERM(DerivationTest, simple, TEST_ATERM(DerivationTest, simple,
R"(Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))",
makeSimpleDrv(*store), makeSimpleDrv(*store),
"simple-derivation") "simple-derivation")
@ -321,45 +332,9 @@ Derivation makeDynDepDerivation(const Store & store) {
return drv; return drv;
} }
TEST_JSON(DynDerivationTest, dynDerivationDeps, TEST_JSON(DynDerivationTest, dynDerivationDeps, makeDynDepDerivation(*store))
R"({
"name": "dyn-dep-derivation",
"inputSrcs": [
"/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"
],
"inputDrvs": {
"/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": {
"dynamicOutputs": {
"cat": {
"dynamicOutputs": {},
"outputs": ["kitten"]
},
"goose": {
"dynamicOutputs": {},
"outputs": ["gosling"]
}
},
"outputs": [
"cat",
"dog"
]
}
},
"system": "wasm-sel4",
"builder": "foo",
"args": [
"bar",
"baz"
],
"env": {
"BIG_BAD": "WOLF"
},
"outputs": {}
})",
makeDynDepDerivation(*store))
TEST_ATERM(DynDerivationTest, dynDerivationDeps, TEST_ATERM(DynDerivationTest, dynDerivationDeps,
R"(DrvWithVersion("xp-dyn-drv",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))",
makeDynDepDerivation(*store), makeDynDepDerivation(*store),
"dyn-dep-derivation") "dyn-dep-derivation")

Some files were not shown because too many files have changed in this diff Show more