max/nix-super
1
0
Fork 0
mirror of https://github.com/privatevoid-net/nix-super.git synced 2024-11-25 23:36:16 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'nixos/master'

Browse source
This commit is contained in:
Max Headroom 2023-10-31 00:05:37 +01:00
commit e5a5fbc0fb
194 changed files with 3847 additions and 1993 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
.github/workflows/ci.yml vendored
View file

@ -101,6 +101,9 @@ jobs:
docker_push_image:
needs: [check_secrets, tests]
permissions:
contents: read
packages: write
if: >-
github.event_name == 'push' &&
github.ref_name == 'master' &&
@ -126,6 +129,9 @@ jobs:
- run: docker load -i ./result/image.tar.gz
- run: docker tag nix:$NIX_VERSION nixos/nix:$NIX_VERSION
- run: docker tag nix:$NIX_VERSION nixos/nix:master
# We'll deploy the newly built image to both Docker Hub and Github Container Registry.
#
# Push to Docker Hub first
- name: Login to Docker Hub
uses: docker/login-action@v3
with:
@ -133,3 +139,20 @@ jobs:
password: ${{ secrets.DOCKERHUB_TOKEN }}
- run: docker push nixos/nix:$NIX_VERSION
- run: docker push nixos/nix:master
# Push to GitHub Container Registry as well
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Push image
run: |
IMAGE_ID=ghcr.io/${{ github.repository_owner }}/nix
# Change all uppercase to lowercase
IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
docker tag nix:$NIX_VERSION $IMAGE_ID:$NIX_VERSION
docker tag nix:$NIX_VERSION $IMAGE_ID:master
docker push $IMAGE_ID:$NIX_VERSION
docker push $IMAGE_ID:master

2
.gitignore vendored
View file

@ -138,7 +138,9 @@ nix-rust/target
result
# IDE
.vscode/
.idea/
# clangd and possibly more
.cache/

5
CONTRIBUTING.md
View file

@ -27,9 +27,12 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy).
1. Search for related issues that cover what you're going to work on.
It could help to mention there that you will work on the issue.
Issues labeled [good first issue](https://github.com/NixOS/nix/labels/good-first-issue) should be relatively easy to fix and are likely to get merged quickly.
Issues labeled [good first issue](https://github.com/NixOS/nix/labels/good%20first%20issue) should be relatively easy to fix and are likely to get merged quickly.
Pull requests addressing issues labeled [idea approved](https://github.com/NixOS/nix/labels/idea%20approved) or [RFC](https://github.com/NixOS/nix/labels/RFC) are especially welcomed by maintainers and will receive prioritised review.
If you are proficient with C++, addressing one of the [popular issues](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc) will be highly appreciated by maintainers and Nix users all over the world.
For far-reaching changes, please investigate possible blockers and design implications, and coordinate with maintainers before investing too much time in writing code that may not end up getting merged.
If there is no relevant issue yet and you're not sure whether your change is likely to be accepted, [open an issue](https://github.com/NixOS/nix/issues/new/choose) yourself.
2. Check for [pull requests](https://github.com/NixOS/nix/pulls) that might already cover the contribution you are about to make.

2
doc/manual/generate-manpage.nix
View file

@ -2,7 +2,7 @@ let
inherit (builtins)
attrNames attrValues fromJSON listToAttrs mapAttrs groupBy
concatStringsSep concatMap length lessThan replaceStrings sort;
inherit (import ./utils.nix) attrsToList concatStrings optionalString filterAttrs trim squash unique;
inherit (import <nix/utils.nix>) attrsToList concatStrings optionalString filterAttrs trim squash unique;
showStoreDocs = import ./generate-store-info.nix;
in

6
doc/manual/local.mk
View file

@ -32,7 +32,7 @@ dummy-env = env -i \
NIX_STATE_DIR=/dummy \
NIX_CONFIG='cores = 0'
nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw
nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix=doc/manual --store dummy:// --impure --raw
# re-implement mdBook's include directive to make it usable for terminal output and for proper @docroot@ substitution
define process-includes
@ -125,7 +125,7 @@ $(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $
@mv $@.tmp $@
$(d)/xp-features.json: $(bindir)/nix
$(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-xp-features > $@.tmp
$(trace-gen) $(dummy-env) $(bindir)/nix __dump-xp-features > $@.tmp
@mv $@.tmp $@
$(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix
@ -141,7 +141,7 @@ $(d)/src/language/builtin-constants.md: $(d)/language.json $(d)/generate-builtin
@mv $@.tmp $@
$(d)/language.json: $(bindir)/nix
$(trace-gen) $(dummy-env) NIX_PATH=nix/corepkgs=corepkgs $(bindir)/nix __dump-language > $@.tmp
$(trace-gen) $(dummy-env) $(bindir)/nix __dump-language > $@.tmp
@mv $@.tmp $@
# Generate the HTML manual.

22
doc/manual/src/SUMMARY.md.in
View file

@ -16,16 +16,8 @@
- [Environment Variables](installation/env-variables.md)
- [Upgrading Nix](installation/upgrading.md)
- [Uninstalling Nix](installation/uninstall.md)
- [Package Management](package-management/package-management.md)
- [Basic Package Management](package-management/basic-package-mgmt.md)
- [Profiles](package-management/profiles.md)
- [Garbage Collection](package-management/garbage-collection.md)
- [Garbage Collector Roots](package-management/garbage-collector-roots.md)
- [Sharing Packages Between Machines](package-management/sharing-packages.md)
- [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md)
- [Copying Closures via SSH](package-management/copy-closure.md)
- [Serving a Nix store via SSH](package-management/ssh-substituter.md)
- [Serving a Nix store via S3](package-management/s3-substituter.md)
- [Nix Store](store/index.md)
- [File System Object](store/file-system-object.md)
- [Nix Language](language/index.md)
- [Data Types](language/values.md)
- [Language Constructs](language/constructs.md)
@ -37,7 +29,16 @@
- [Import From Derivation](language/import-from-derivation.md)
- [Built-in Constants](language/builtin-constants.md)
- [Built-in Functions](language/builtins.md)
- [Package Management](package-management/package-management.md)
- [Profiles](package-management/profiles.md)
- [Garbage Collection](package-management/garbage-collection.md)
- [Garbage Collector Roots](package-management/garbage-collector-roots.md)
- [Advanced Topics](advanced-topics/advanced-topics.md)
- [Sharing Packages Between Machines](package-management/sharing-packages.md)
- [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md)
- [Copying Closures via SSH](package-management/copy-closure.md)
- [Serving a Nix store via SSH](package-management/ssh-substituter.md)
- [Serving a Nix store via S3](package-management/s3-substituter.md)
- [Remote Builds](advanced-topics/distributed-builds.md)
- [Tuning Cores and Jobs](advanced-topics/cores-vs-jobs.md)
- [Verifying Build Reproducibility](advanced-topics/diff-hook.md)
@ -99,7 +100,6 @@
- [Channels](command-ref/files/channels.md)
- [Default Nix expression](command-ref/files/default-nix-expression.md)
- [Architecture and Design](architecture/architecture.md)
- [File System Object](architecture/file-system-object.md)
- [Protocols](protocols/protocols.md)
- [Serving Tarball Flakes](protocols/tarball-fetcher.md)
- [Derivation "ATerm" file format](protocols/derivation-aterm.md)

4
doc/manual/src/advanced-topics/distributed-builds.md
View file

@ -12,14 +12,14 @@ machine is accessible via SSH and that it has Nix installed. You can
test whether connecting to the remote Nix instance works, e.g.
```console
$ nix store ping --store ssh://mac
$ nix store info --store ssh://mac
```
will try to connect to the machine named `mac`. It is possible to
specify an SSH identity file as part of the remote store URI, e.g.
```console
$ nix store ping --store ssh://mac?ssh-key=/home/alice/my-key
$ nix store info --store ssh://mac?ssh-key=/home/alice/my-key
```
Since builds should be non-interactive, the key should not have a

5
doc/manual/src/advanced-topics/post-build-hook.md
View file

@ -17,9 +17,8 @@ the build loop.
# Prerequisites
This tutorial assumes you have [configured an S3-compatible binary
cache](../package-management/s3-substituter.md), and that the `root`
user's default AWS profile can upload to the bucket.
This tutorial assumes you have configured an [S3-compatible binary cache](@docroot@/command-ref/new-cli/nix3-help-stores.md#s3-binary-cache-store) as a [substituter](../command-ref/conf-file.md#conf-substituters),
and that the `root` user's default AWS profile can upload to the bucket.
# Set up a Signing Key

1
doc/manual/src/architecture/architecture.md
View file

@ -59,6 +59,7 @@ The [Nix language](../language/index.md) evaluator transforms Nix expressions in
The command line interface and Nix expressions are what users deal with most.
> **Note**
>
> The Nix language itself does not have a notion of *packages* or *configurations*.
> As far as we are concerned here, the inputs and results of a build plan are just data.

6
doc/manual/src/command-ref/nix-shell.md
View file

@ -235,14 +235,14 @@ package like Terraform:
```bash
#! /usr/bin/env nix-shell
#! nix-shell -i bash --packages "terraform.withPlugins (plugins: [ plugins.openstack ])"
#! nix-shell -i bash --packages 'terraform.withPlugins (plugins: [ plugins.openstack ])'
terraform apply
```
> **Note**
>
> You must use double quotes (`"`) when passing a simple Nix expression
> You must use single or double quotes (`'`, `"`) when passing a simple Nix expression
> in a nix-shell shebang.
Finally, using the merging of multiple nix-shell shebangs the following
@ -251,7 +251,7 @@ branch):
```haskell
#! /usr/bin/env nix-shell
#! nix-shell -i runghc --packages "haskellPackages.ghcWithPackages (ps: [ps.download-curl ps.tagsoup])"
#! nix-shell -i runghc --packages 'haskellPackages.ghcWithPackages (ps: [ps.download-curl ps.tagsoup])'
#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/nixos-20.03.tar.gz
import Network.Curl.Download

11
doc/manual/src/contributing/documentation.md
View file

@ -73,6 +73,17 @@ It should therefore aim to be correct, consistent, complete, and easy to navigat
Non-trivial examples may need additional explanation, especially if they use concepts from outside the given context.
- Always explain code examples in the text.
Use comments in code samples very sparingly, for instance to highlight a particular aspect.
Readers tend to glance over large amounts of code when scanning for information.
Especially beginners will likely find reading more complex-looking code strenuous and may therefore avoid it altogether.
If a code sample appears to require a lot of inline explanation, consider replacing it with a simpler one.
If that's not possible, break the example down into multiple parts, explain them separately, and then show the combined result at the end.
This should be a last resort, as that would amount to writing a [tutorial](https://diataxis.fr/tutorials/) on the given subject.
- Use British English.
This is a somewhat arbitrary choice to force consistency, and accounts for the fact that a majority of Nix users and developers are from Europe.

31
doc/manual/src/glossary.md
View file

@ -58,22 +58,16 @@
- [store]{#gloss-store}
The location in the file system where store objects live. Typically
`/nix/store`.
A collection of store objects, with operations to manipulate that collection.
See [Nix Store] for details.
From the perspective of the location where Nix is
invoked, the Nix store can be referred to
as a "_local_" or a "_remote_" one:
There are many types of stores.
See [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md) for a complete list.
+ A [local store]{#gloss-local-store} exists on the filesystem of
the machine where Nix is invoked. You can use other
local stores by passing the `--store` flag to the
`nix` command. Local stores can be used for building derivations.
+ A *remote store* exists anywhere other than the
local filesystem. One example is the `/nix/store`
directory on another machine, accessed via `ssh` or
served by the `nix-serve` Perl script.
From the perspective of the location where Nix is invoked, the Nix store can be referred to _local_ or _remote_.
Only a [local store]{#gloss-local-store} exposes a location in the file system of the machine where Nix is invoked that allows access to store objects, typically `/nix/store`.
Local stores can be used for building [derivations](#derivation).
See [Local Store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) for details.
[store]: #gloss-store
[local store]: #gloss-local-store
@ -103,15 +97,19 @@
The Nix data model for representing simplified file system data.
See [File System Object](@docroot@/architecture/file-system-object.md) for details.
See [File System Object](@docroot@/store/file-system-object.md) for details.
[file system object]: #gloss-file-system-object
- [store object]{#gloss-store-object}
A store object consists of a [file system object], [reference]s to other store objects, and other metadata.
Part of the contents of a [store].
A store object consists of a [file system object], [references][reference] to other store objects, and other metadata.
It can be referred to by a [store path].
See [Store Object](@docroot@/store/index.md#store-object) for details.
[store object]: #gloss-store-object
- [IFD]{#gloss-ifd}
@ -207,6 +205,7 @@
- [output]{#gloss-output}
A [store object] produced by a [derivation].
See [the `outputs` argument to the `derivation` function](@docroot@/language/derivations.md#attr-outputs) for details.
[output]: #gloss-output

8
doc/manual/src/installation/installing-docker.md
View file

@ -3,14 +3,14 @@
To run the latest stable release of Nix with Docker run the following command:
```console
$ docker run -ti nixos/nix
Unable to find image 'nixos/nix:latest' locally
latest: Pulling from nixos/nix
$ docker run -ti ghcr.io/nixos/nix
Unable to find image 'ghcr.io/nixos/nix:latest' locally
latest: Pulling from ghcr.io/nixos/nix
5843afab3874: Pull complete
b52bf13f109c: Pull complete
1e2415612aa3: Pull complete
Digest: sha256:27f6e7f60227e959ee7ece361f75d4844a40e1cc6878b6868fe30140420031ff
Status: Downloaded newer image for nixos/nix:latest
Status: Downloaded newer image for ghcr.io/nixos/nix:latest
35ca4ada6e96:/# nix --version
nix (Nix) 2.3.12
35ca4ada6e96:/# exit

214
doc/manual/src/language/derivations.md
View file

@ -8,8 +8,6 @@ It outputs an attribute set, and produces a [store derivation] as a side effect
[store derivation]: @docroot@/glossary.md#gloss-store-derivation
<!-- FIXME: add a section on output attributes -->
## Input attributes
### Required
@ -17,11 +15,22 @@ It outputs an attribute set, and produces a [store derivation] as a side effect
- [`name`]{#attr-name} ([String](@docroot@/language/values.md#type-string))
A symbolic name for the derivation.
It is added to the [store derivation]'s [path](@docroot@/glossary.md#gloss-store-path) and its [output paths][output path].
It is added to the [store path] of the corresponding [store derivation] as well as to its [output paths](@docroot@/glossary.md#gloss-output-path).
Example: `name = "hello";`
[store path]: @docroot@/glossary.md#gloss-store-path
> **Example**
>
> ```nix
> derivation {
> name = "hello";
> # ...
> }
> ```
>
> The store derivation's path will be `/nix/store/<hash>-hello.drv`.
> The [output](#attr-outputs) paths will be of the form `/nix/store/<hash>-hello[-<output>]`
The store derivation's path will be `/nix/store/<hash>-hello.drv`, and the output paths will be of the form `/nix/store/<hash>-hello[-<output>]`
- [`system`]{#attr-system} ([String](@docroot@/language/values.md#type-string))
The system type on which the [`builder`](#attr-builder) executable is meant to be run.
@ -29,77 +38,175 @@ It outputs an attribute set, and produces a [store derivation] as a side effect
A necessary condition for Nix to build derivations locally is that the `system` attribute matches the current [`system` configuration option].
It can automatically [build on other platforms](../advanced-topics/distributed-builds.md) by forwarding build requests to other machines.
Examples:
`system = "x86_64-linux";`
`system = builtins.currentSystem;`
[`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation.
[`system` configuration option]: @docroot@/command-ref/conf-file.md#conf-system
> **Example**
>
> Declare a derivation to be built on a specific system type:
>
> ```nix
> derivation {
> # ...
> system = "x86_64-linux";
> # ...
> }
> ```
> **Example**
>
> Declare a derivation to be built on the system type that evaluates the expression:
>
> ```nix
> derivation {
> # ...
> system = builtins.currentSystem;
> # ...
> }
> ```
>
> [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation.
- [`builder`]{#attr-builder} ([Path](@docroot@/language/values.md#type-path) | [String](@docroot@/language/values.md#type-string))
Path to an executable that will perform the build.
Examples:
> **Example**
>
> Use the file located at `/bin/bash` as the builder executable:
>
> ```nix
> derivation {
> # ...
> builder = "/bin/bash";
> # ...
> };
> ```
`builder = "/bin/bash";`
<!-- -->
`builder = ./builder.sh;`
> **Example**
>
> Copy a local file to the Nix store for use as the builder executable:
>
> ```nix
> derivation {
> # ...
> builder = ./builder.sh;
> # ...
> };
> ```
`builder = "${pkgs.python}/bin/python";`
<!-- -->
> **Example**
>
> Use a file from another derivation as the builder executable:
>
> ```nix
> let pkgs = import <nixpkgs> {}; in
> derivation {
> # ...
> builder = "${pkgs.python}/bin/python";
> # ...
> };
> ```
### Optional
- [`args`]{#attr-args} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) Default: `[ ]`
- [`args`]{#attr-args} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string))
Default: `[ ]`
Command-line arguments to be passed to the [`builder`](#attr-builder) executable.
Example: `args = [ "-c" "echo hello world > $out" ];`
> **Example**
>
> Pass arguments to Bash to interpret a shell command:
>
> ```nix
> derivation {
> # ...
> builder = "/bin/bash";
> args = [ "-c" "echo hello world > $out" ];
> # ...
> };
> ```
- [`outputs`]{#attr-outputs} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) Default: `[ "out" ]`
- [`outputs`]{#attr-outputs} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string))
Default: `[ "out" ]`
Symbolic outputs of the derivation.
Each output name is passed to the [`builder`](#attr-builder) executable as an environment variable with its value set to the corresponding [output path].
Each output name is passed to the [`builder`](#attr-builder) executable as an environment variable with its value set to the corresponding [store path].
[output path]: @docroot@/glossary.md#gloss-output-path
By default, a derivation produces a single output path called `out`.
However, derivations can produce multiple output paths.
By default, a derivation produces a single output called `out`.
However, derivations can produce multiple outputs.
This allows the associated [store objects](@docroot@/glossary.md#gloss-store-object) and their [closures](@docroot@/glossary.md#gloss-closure) to be copied or garbage-collected separately.
Examples:
> **Example**
>
> Imagine a library package that provides a dynamic library, header files, and documentation.
> A program that links against such a library doesn’t need the header files and documentation at runtime, and it doesn’t need the documentation at build time.
> Thus, the library package could specify:
>
> ```nix
> derivation {
> # ...
> outputs = [ "lib" "dev" "doc" ];
> # ...
> }
> ```
>
> This will cause Nix to pass environment variables `lib`, `dev`, and `doc` to the builder containing the intended store paths of each output.
> The builder would typically do something like
>
> ```bash
> ./configure \
> --libdir=$lib/lib \
> --includedir=$dev/include \
> --docdir=$doc/share/doc
> ```
>
> for an Autoconf-style package.
Imagine a library package that provides a dynamic library, header files, and documentation.
A program that links against such a library doesn’t need the header files and documentation at runtime, and it doesn’t need the documentation at build time.
Thus, the library package could specify:
The name of an output is combined with the name of the derivation to create the name part of the output's store path, unless it is `out`, in which case just the name of the derivation is used.
```nix
derivation {
# ...
outputs = [ "lib" "dev" "doc" ];
# ...
}
```
> **Example**
>
>
> ```nix
> derivation {
> name = "example";
> outputs = [ "lib" "dev" "doc" "out" ];
> # ...
> }
> ```
>
> The store derivation path will be `/nix/store/<hash>-example.drv`.
> The output paths will be
> - `/nix/store/<hash>-example-lib`
> - `/nix/store/<hash>-example-dev`
> - `/nix/store/<hash>-example-doc`
> - `/nix/store/<hash>-example`
This will cause Nix to pass environment variables `lib`, `dev`, and `doc` to the builder containing the intended store paths of each output.
The builder would typically do something like
You can refer to each output of a derivation by selecting it as an attribute.
The first element of `outputs` determines the *default output* and ends up at the top-level.
```bash
./configure \
--libdir=$lib/lib \
--includedir=$dev/include \
--docdir=$doc/share/doc
```
for an Autoconf-style package.
You can refer to each output of a derivation by selecting it as an attribute, e.g. `myPackage.lib` or `myPackage.doc`.
The first element of `outputs` determines the *default output*.
Therefore, in the given example, `myPackage` is equivalent to `myPackage.lib`.
> **Example**
>
> Select an output by attribute name:
>
> ```nix
> let
> myPackage = derivation {
> name = "example";
> outputs = [ "lib" "dev" "doc" "out" ];
> # ...
> };
> in myPackage.dev
> ```
>
> Since `lib` is the first output, `myPackage` is equivalent to `myPackage.lib`.
<!-- FIXME: refer to the output attributes when we have one -->
@ -123,8 +230,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect
reside in the Nix store.
- A *derivation* causes that derivation to be built prior to the
present derivation; its default output path is put in the
environment variable.
present derivation. The environment variable is set to the [store path] of the derivation's default [output](#attr-outputs).
- Lists of the previous types are also allowed. They are simply
concatenated, separated by spaces.
@ -132,6 +238,8 @@ It outputs an attribute set, and produces a [store derivation] as a side effect
- `true` is passed as the string `1`, `false` and `null` are
passed as an empty string.
<!-- FIXME: add a section on output attributes -->
## Builder execution
The [`builder`](#attr-builder) is executed as follows:

4
doc/manual/src/language/operators.md
View file

@ -59,10 +59,14 @@ An attribute path is a dot-separated list of [attribute names](./values.md#attri
Test whether [attribute set] *attrset* contains the attribute denoted by *attrpath*.
The result is a [Boolean] value.
See also: [`builtins.hasAttr`](@docroot@/language/builtins.md#builtins-hasAttr)
[Boolean]: ./values.md#type-boolean
[Has attribute]: #has-attribute
After evaluating *attrset* and *attrpath*, the computational complexity is O(log(*n*)) for *n* attributes in the *attrset*
## Arithmetic
Numbers are type-compatible:

179
doc/manual/src/package-management/basic-package-mgmt.md
View file

@ -1,179 +0,0 @@
# Basic Package Management
The main command for package management is
[`nix-env`](../command-ref/nix-env.md). You can use it to install,
upgrade, and erase packages, and to query what packages are installed
or are available for installation.
In Nix, different users can have different “views” on the set of
installed applications. That is, there might be lots of applications
present on the system (possibly in many different versions), but users
can have a specific selection of those active — where “active” just
means that it appears in a directory in the user’s `PATH`. Such a view
on the set of installed applications is called a *user environment*,
which is just a directory tree consisting of symlinks to the files of
the active applications.
Components are installed from a set of *Nix expressions* that tell Nix
how to build those packages, including, if necessary, their
dependencies. There is a collection of Nix expressions called the
Nixpkgs package collection that contains packages ranging from basic
development stuff such as GCC and Glibc, to end-user applications like
Mozilla Firefox. (Nix is however not tied to the Nixpkgs package
collection; you could write your own Nix expressions based on Nixpkgs,
or completely new ones.)
You can manually download the latest version of Nixpkgs from
<https://github.com/NixOS/nixpkgs>. However, it’s much more
convenient to use the Nixpkgs [*channel*](../command-ref/nix-channel.md), since it makes
it easy to stay up to date with new versions of Nixpkgs. Nixpkgs is
automatically added to your list of “subscribed” channels when you
install Nix. If this is not the case for some reason, you can add it
as follows:
```console
$ nix-channel --add https://nixos.org/channels/nixpkgs-unstable
$ nix-channel --update
```
> **Note**
>
> On NixOS, you’re automatically subscribed to a NixOS channel
> corresponding to your NixOS major release (e.g.
> <http://nixos.org/channels/nixos-21.11>). A NixOS channel is identical
> to the Nixpkgs channel, except that it contains only Linux binaries
> and is updated only if a set of regression tests succeed.
You can view the set of available packages in Nixpkgs:
```console
$ nix-env --query --available --attr-path
nixpkgs.aterm aterm-2.2
nixpkgs.bash bash-3.0
nixpkgs.binutils binutils-2.15
nixpkgs.bison bison-1.875d
nixpkgs.blackdown blackdown-1.4.2
nixpkgs.bzip2 bzip2-1.0.2
…
```
The flag `-q` specifies a query operation, `-a` means that you want
to show the “available” (i.e., installable) packages, as opposed to the
installed packages, and `-P` prints the attribute paths that can be used
to unambiguously select a package for installation (listed in the first column).
If you downloaded Nixpkgs yourself, or if you checked it out from GitHub,
then you need to pass the path to your Nixpkgs tree using the `-f` flag:
```console
$ nix-env --query --available --attr-path --file /path/to/nixpkgs
aterm aterm-2.2
bash bash-3.0
…
```
where */path/to/nixpkgs* is where you’ve unpacked or checked out
Nixpkgs.
You can filter the packages by name:
```console
$ nix-env --query --available --attr-path firefox
nixpkgs.firefox-esr firefox-91.3.0esr
nixpkgs.firefox firefox-94.0.1
```
and using regular expressions:
```console
$ nix-env --query --available --attr-path 'firefox.*'
```
It is also possible to see the *status* of available packages, i.e.,
whether they are installed into the user environment and/or present in
the system:
```console
$ nix-env --query --available --attr-path --status
…
-PS nixpkgs.bash bash-3.0
--S nixpkgs.binutils binutils-2.15
IPS nixpkgs.bison bison-1.875d
…
```
The first character (`I`) indicates whether the package is installed in
your current user environment. The second (`P`) indicates whether it is
present on your system (in which case installing it into your user
environment would be a very quick operation). The last one (`S`)
indicates whether there is a so-called *substitute* for the package,
which is Nix’s mechanism for doing binary deployment. It just means that
Nix knows that it can fetch a pre-built package from somewhere
(typically a network server) instead of building it locally.
You can install a package using `nix-env --install --attr `. For instance,
```console
$ nix-env --install --attr nixpkgs.subversion
```
will install the package called `subversion` from `nixpkgs` channel (which is, of course, the
[Subversion version management system](http://subversion.tigris.org/)).
> **Note**
>
> When you ask Nix to install a package, it will first try to get it in
> pre-compiled form from a *binary cache*. By default, Nix will use the
> binary cache <https://cache.nixos.org>; it contains binaries for most
> packages in Nixpkgs. Only if no binary is available in the binary
> cache, Nix will build the package from source. So if `nix-env
> -iA nixpkgs.subversion` results in Nix building stuff from source, then either
> the package is not built for your platform by the Nixpkgs build
> servers, or your version of Nixpkgs is too old or too new. For
> instance, if you have a very recent checkout of Nixpkgs, then the
> Nixpkgs build servers may not have had a chance to build everything
> and upload the resulting binaries to <https://cache.nixos.org>. The
> Nixpkgs channel is only updated after all binaries have been uploaded
> to the cache, so if you stick to the Nixpkgs channel (rather than
> using a Git checkout of the Nixpkgs tree), you will get binaries for
> most packages.
Naturally, packages can also be uninstalled. Unlike when installing, you will
need to use the derivation name (though the version part can be omitted),
instead of the attribute path, as `nix-env` does not record which attribute
was used for installing:
```console
$ nix-env --uninstall subversion
```
Upgrading to a new version is just as easy. If you have a new release of
Nix Packages, you can do:
```console
$ nix-env --upgrade --attr nixpkgs.subversion
```
This will *only* upgrade Subversion if there is a “newer” version in the
new set of Nix expressions, as defined by some pretty arbitrary rules
regarding ordering of version numbers (which generally do what you’d
expect of them). To just unconditionally replace Subversion with
whatever version is in the Nix expressions, use `-i` instead of `-u`;
`-i` will remove whatever version is already installed.
You can also upgrade all packages for which there are newer versions:
```console
$ nix-env --upgrade
```
Sometimes it’s useful to be able to ask what `nix-env` would do, without
actually doing it. For instance, to find out what packages would be
upgraded by `nix-env --upgrade `, you can do
```console
$ nix-env --upgrade --dry-run
(dry run; not doing anything)
upgrading `libxslt-1.1.0' to `libxslt-1.1.10'
upgrading `graphviz-1.10' to `graphviz-1.12'
upgrading `coreutils-5.0' to `coreutils-5.2.1'
```

115
doc/manual/src/package-management/s3-substituter.md
View file

@ -1,115 +0,0 @@
# Serving a Nix store via S3
Nix has [built-in support](@docroot@/command-ref/new-cli/nix3-help-stores.md#s3-binary-cache-store)
for storing and fetching store paths from
Amazon S3 and S3-compatible services. This uses the same *binary*
cache mechanism that Nix usually uses to fetch prebuilt binaries from
[cache.nixos.org](https://cache.nixos.org/).
In this example we will use the bucket named `example-nix-cache`.
## Anonymous Reads to your S3-compatible binary cache
If your binary cache is publicly accessible and does not require
authentication, the simplest and easiest way to use Nix with your S3
compatible binary cache is to use the HTTP URL for that cache.
For AWS S3 the binary cache URL for example bucket will be exactly
<https://example-nix-cache.s3.amazonaws.com> or
<s3://example-nix-cache>. For S3 compatible binary caches, consult that
cache's documentation.
Your bucket will need the following bucket policy:
```json
{
"Id": "DirectReads",
"Version": "2012-10-17",
"Statement": [
{
"Sid": "AllowDirectReads",
"Action": [
"s3:GetObject",
"s3:GetBucketLocation"
],
"Effect": "Allow",
"Resource": [
"arn:aws:s3:::example-nix-cache",
"arn:aws:s3:::example-nix-cache/*"
],
"Principal": "*"
}
]
}
```
## Authenticated Reads to your S3 binary cache
For AWS S3 the binary cache URL for example bucket will be exactly
<s3://example-nix-cache>.
Nix will use the [default credential provider
chain](https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html)
for authenticating requests to Amazon S3.
Nix supports authenticated reads from Amazon S3 and S3 compatible binary
caches.
Your bucket will need a bucket policy allowing the desired users to
perform the `s3:GetObject` and `s3:GetBucketLocation` action on all
objects in the bucket. The [anonymous policy given
above](#anonymous-reads-to-your-s3-compatible-binary-cache) can be
updated to have a restricted `Principal` to support this.
## Authenticated Writes to your S3-compatible binary cache
Nix support fully supports writing to Amazon S3 and S3 compatible
buckets. The binary cache URL for our example bucket will be
<s3://example-nix-cache>.
Nix will use the [default credential provider
chain](https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html)
for authenticating requests to Amazon S3.
Your account will need the following IAM policy to upload to the cache:
```json
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "UploadToCache",
"Effect": "Allow",
"Action": [
"s3:AbortMultipartUpload",
"s3:GetBucketLocation",
"s3:GetObject",
"s3:ListBucket",
"s3:ListBucketMultipartUploads",
"s3:ListMultipartUploadParts",
"s3:PutObject"
],
"Resource": [
"arn:aws:s3:::example-nix-cache",
"arn:aws:s3:::example-nix-cache/*"
]
}
]
}
```
## Examples
To upload with a specific credential profile for Amazon S3:
```console
$ nix copy nixpkgs.hello \
--to 's3://example-nix-cache?profile=cache-upload&region=eu-west-2'
```
To upload to an S3-compatible binary cache:
```console
$ nix copy nixpkgs.hello --to \
's3://example-nix-cache?profile=cache-upload&scheme=https&endpoint=minio.example.com'
```

8
doc/manual/src/release-notes/rl-next.md
View file

@ -7,3 +7,11 @@
- The experimental feature `repl-flake` is no longer needed, as its functionality is now part of the `flakes` experimental feature. To get the previous behavior, use the `--file/--expr` flags accordingly.
- Introduce new flake installable syntax `flakeref#.attrPath` where the "." prefix denotes no searching of default attribute prefixes like `packages.<SYSTEM>` or `legacyPackages.<SYSTEM>`.
- Nix adds `apple-virt` to the default system features on macOS systems that support virtualization. This is similar to what's done for the `kvm` system feature on Linux hosts.
- Introduce a new built-in function [`builtins.convertHash`](@docroot@/language/builtins.md#builtins-convertHash).
- `nix-shell` shebang lines now support single-quoted arguments.
- `builtins.fetchTree` is now marked as stable.

0
doc/manual/src/architecture/file-system-object.md → doc/manual/src/store/file-system-object.md
View file

4
doc/manual/src/store/index.md Normal file
View file

@ -0,0 +1,4 @@
# Nix Store
The *Nix store* is an abstraction used by Nix to store immutable filesystem artifacts (such as software packages) that can have dependencies (*references*) between them.
There are multiple implementations of the Nix store, such as the actual filesystem (`/nix/store`) and binary caches.

44
flake.nix
View file

@ -474,47 +474,13 @@
hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie";
passthru.perl-bindings = with final; perl.pkgs.toPerlModule (currentStdenv.mkDerivation {
name = "nix-super-perl-${version}";
src = fileset.toSource {
root = ./.;
fileset = fileset.intersect baseFiles (fileset.unions [
./perl
./.version
./m4
./mk
]);
};
nativeBuildInputs =
[ buildPackages.autoconf-archive
buildPackages.autoreconfHook
buildPackages.pkg-config
];
buildInputs =
[ nix
curl
bzip2
xz
pkgs.perl
boost
]
++ lib.optional (currentStdenv.isLinux || currentStdenv.isDarwin) libsodium
++ lib.optional currentStdenv.isDarwin darwin.apple_sdk.frameworks.Security;
configureFlags = [
"--with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix}"
"--with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix}"
];
enableParallelBuilding = true;
postUnpack = "sourceRoot=$sourceRoot/perl";
});
passthru.perl-bindings = final.callPackage ./perl {
inherit fileset;
stdenv = currentStdenv;
};
meta.platforms = lib.platforms.unix;
meta.mainProgram = "nix";
});
lowdown-nix = with final; currentStdenv.mkDerivation rec {

4
maintainers/README.md
View file

@ -50,7 +50,9 @@ The team meets twice a week:
1. Code review on pull requests from [In review](#in-review).
2. Other chores and tasks.
Meeting notes are collected on a [collaborative scratchpad](https://pad.lassul.us/Cv7FpYx-Ri-4VjUykQOLAw), and published on Discourse under the [Nix category](https://discourse.nixos.org/c/dev/nix/50).
Meeting notes are collected on a [collaborative scratchpad](https://pad.lassul.us/Cv7FpYx-Ri-4VjUykQOLAw).
Notes on issues and pull requests are posted as comments and linked from the meeting notes, so they are easy to find from both places.
[All meeting notes](https://discourse.nixos.org/search?expanded=true&q=Nix%20team%20meeting%20minutes%20%23%20%23dev%3Anix%20in%3Atitle%20order%3Alatest_topic) are published on Discourse under the [Nix category](https://discourse.nixos.org/c/dev/nix/50).
## Project board protocol

51
perl/default.nix Normal file
View file

@ -0,0 +1,51 @@
{ lib, fileset
, stdenv
, perl, perlPackages
, autoconf-archive, autoreconfHook, pkg-config
, nix, curl, bzip2, xz, boost, libsodium, darwin
}:
perl.pkgs.toPerlModule (stdenv.mkDerivation {
name = "nix-perl-${nix.version}";
src = fileset.toSource {
root = ../.;
fileset = fileset.unions [
../.version
../m4
../mk
./MANIFEST
./Makefile
./Makefile.config.in
./configure.ac
./lib
./local.mk
];
};
nativeBuildInputs =
[ autoconf-archive
autoreconfHook
pkg-config
];
buildInputs =
[ nix
curl
bzip2
xz
perl
boost
]
++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security;
configureFlags = [
"--with-dbi=${perlPackages.DBI}/${perl.libPrefix}"
"--with-dbd-sqlite=${perlPackages.DBDSQLite}/${perl.libPrefix}"
];
enableParallelBuilding = true;
postUnpack = "sourceRoot=$sourceRoot/perl";
})

12
perl/lib/Nix/Store.xs
View file

@ -78,7 +78,7 @@ SV * queryReferences(char * path)
SV * queryPathHash(char * path)
PPCODE:
try {
auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base32, true);
auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(HashFormat::Base32, true);
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
} catch (Error & e) {
croak("%s", e.what());
@ -104,7 +104,7 @@ SV * queryPathInfo(char * path, int base32)
XPUSHs(&PL_sv_undef);
else
XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0)));
auto s = info->narHash.to_string(base32 ? Base32 : Base16, true);
auto s = info->narHash.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, true);
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
mXPUSHi(info->registrationTime);
mXPUSHi(info->narSize);
@ -206,7 +206,7 @@ SV * hashPath(char * algo, int base32, char * path)
PPCODE:
try {
Hash h = hashPath(parseHashType(algo), path).first;
auto s = h.to_string(base32 ? Base32 : Base16, false);
auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false);
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
} catch (Error & e) {
croak("%s", e.what());
@ -217,7 +217,7 @@ SV * hashFile(char * algo, int base32, char * path)
PPCODE:
try {
Hash h = hashFile(parseHashType(algo), path);
auto s = h.to_string(base32 ? Base32 : Base16, false);
auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false);
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
} catch (Error & e) {
croak("%s", e.what());
@ -228,7 +228,7 @@ SV * hashString(char * algo, int base32, char * s)
PPCODE:
try {
Hash h = hashString(parseHashType(algo), s);
auto s = h.to_string(base32 ? Base32 : Base16, false);
auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false);
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
} catch (Error & e) {
croak("%s", e.what());
@ -239,7 +239,7 @@ SV * convertHash(char * algo, char * s, int toBase32)
PPCODE:
try {
auto h = Hash::parseAny(s, parseHashType(algo));
auto s = h.to_string(toBase32 ? Base32 : Base16, false);
auto s = h.to_string(toBase32 ? HashFormat::Base32 : HashFormat::Base16, false);
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
} catch (Error & e) {
croak("%s", e.what());

104
src/libcmd/command.hh
View file

@ -35,21 +35,28 @@ struct NixMultiCommand : virtual MultiCommand, virtual Command
// For the overloaded run methods
#pragma GCC diagnostic ignored "-Woverloaded-virtual"
/* A command that requires a Nix store. */
/**
* A command that requires a \ref Store "Nix store".
*/
struct StoreCommand : virtual Command
{
StoreCommand();
void run() override;
ref<Store> getStore();
virtual ref<Store> createStore();
/**
* Main entry point, with a `Store` provided
*/
virtual void run(ref<Store>) = 0;
private:
std::shared_ptr<Store> _store;
};
/* A command that copies something between `--from` and `--to`
stores. */
/**
* A command that copies something between `--from` and `--to` \ref
* Store stores.
*/
struct CopyCommand : virtual StoreCommand
{
std::string srcUri, dstUri;
@ -61,6 +68,9 @@ struct CopyCommand : virtual StoreCommand
ref<Store> getDstStore();
};
/**
* A command that needs to evaluate Nix language expressions.
*/
struct EvalCommand : virtual StoreCommand, MixEvalArgs
{
bool startReplOnEvalErrors = false;
@ -80,20 +90,26 @@ private:
std::shared_ptr<EvalState> evalState;
};
/**
* A mixin class for commands that process flakes, adding a few standard
* flake-related options/flags.
*/
struct MixFlakeOptions : virtual Args, EvalCommand
{
flake::LockFlags lockFlags;
std::optional<std::string> needsFlakeInputCompletion = {};
MixFlakeOptions();
virtual std::vector<std::string> getFlakesForCompletion()
/**
* The completion for some of these flags depends on the flake(s) in
* question.
*
* This method should be implemented to gather all flakerefs the
* command is operating with (presumably specified via some other
* arguments) so that the completions for these flags can use them.
*/
virtual std::vector<FlakeRef> getFlakeRefsForCompletion()
{ return {}; }
void completeFlakeInput(std::string_view prefix);
void completionHook() override;
};
struct SourceExprCommand : virtual Args, MixFlakeOptions
@ -129,15 +145,35 @@ struct SourceExprCommand : virtual Args, MixFlakeOptions
virtual Strings getDefaultFlakeAttrPathPrefixes();
void completeInstallable(std::string_view prefix);
/**
* Complete an installable from the given prefix.
*/
void completeInstallable(AddCompletions & completions, std::string_view prefix);
/**
* Convenience wrapper around the underlying function to make setting the
* callback easier.
*/
CompleterClosure getCompleteInstallable();
};
/**
* A mixin class for commands that need a read-only flag.
*
* What exactly is "read-only" is unspecified, but it will usually be
* the \ref Store "Nix store".
*/
struct MixReadOnlyOption : virtual Args
{
MixReadOnlyOption();
};
/* Like InstallablesCommand but the installables are not loaded */
/**
* Like InstallablesCommand but the installables are not loaded.
*
* This is needed by `CmdRepl` which wants to load (and reload) the
* installables itself.
*/
struct RawInstallablesCommand : virtual Args, SourceExprCommand
{
RawInstallablesCommand();
@ -146,19 +182,22 @@ struct RawInstallablesCommand : virtual Args, SourceExprCommand
void run(ref<Store> store) override;
// FIXME make const after CmdRepl's override is fixed up
// FIXME make const after `CmdRepl`'s override is fixed up
virtual void applyDefaultInstallables(std::vector<std::string> & rawInstallables);
bool readFromStdIn = false;
std::vector<std::string> getFlakesForCompletion() override;
std::vector<FlakeRef> getFlakeRefsForCompletion() override;
private:
std::vector<std::string> rawInstallables;
};
/* A command that operates on a list of "installables", which can be
store paths, attribute paths, Nix expressions, etc. */
/**
* A command that operates on a list of "installables", which can be
* store paths, attribute paths, Nix expressions, etc.
*/
struct InstallablesCommand : RawInstallablesCommand
{
virtual void run(ref<Store> store, Installables && installables) = 0;
@ -166,7 +205,9 @@ struct InstallablesCommand : RawInstallablesCommand
void run(ref<Store> store, std::vector<std::string> && rawInstallables) override;
};
/* A command that operates on exactly one "installable" */
/**
* A command that operates on exactly one "installable".
*/
struct InstallableCommand : virtual Args, SourceExprCommand
{
InstallableCommand();
@ -175,10 +216,7 @@ struct InstallableCommand : virtual Args, SourceExprCommand
void run(ref<Store> store) override;
std::vector<std::string> getFlakesForCompletion() override
{
return {_installable};
}
std::vector<FlakeRef> getFlakeRefsForCompletion() override;
private:
@ -192,7 +230,12 @@ struct MixOperateOnOptions : virtual Args
MixOperateOnOptions();
};
/* A command that operates on zero or more store paths. */
/**
* A command that operates on zero or more extant store paths.
*
* If the argument the user passes is a some sort of recipe for a path
* not yet built, it must be built first.
*/
struct BuiltPathsCommand : InstallablesCommand, virtual MixOperateOnOptions
{
private:
@ -224,7 +267,9 @@ struct StorePathsCommand : public BuiltPathsCommand
void run(ref<Store> store, BuiltPaths && paths) override;
};
/* A command that operates on exactly one store path. */
/**
* A command that operates on exactly one store path.
*/
struct StorePathCommand : public StorePathsCommand
{
virtual void run(ref<Store> store, const StorePath & storePath) = 0;
@ -232,7 +277,9 @@ struct StorePathCommand : public StorePathsCommand
void run(ref<Store> store, StorePaths && storePaths) override;
};
/* A helper class for registering commands globally. */
/**
* A helper class for registering \ref Command commands globally.
*/
struct RegisterCommand
{
typedef std::map<std::vector<std::string>, std::function<ref<Command>()>> Commands;
@ -288,7 +335,11 @@ struct MixEnvironment : virtual Args {
MixEnvironment();
/* Modify global environ based on ignoreEnvironment, keep, and unset. It's expected that exec will be called before this class goes out of scope, otherwise environ will become invalid. */
/***
* Modify global environ based on `ignoreEnvironment`, `keep`, and
* `unset`. It's expected that exec will be called before this class
* goes out of scope, otherwise `environ` will become invalid.
*/
void setEnviron();
};
@ -324,9 +375,10 @@ public:
const std::string selfCommandName;
};
void completeFlakeRef(ref<Store> store, std::string_view prefix);
void completeFlakeRef(AddCompletions & completions, ref<Store> store, std::string_view prefix);
void completeFlakeRefWithFragment(
AddCompletions & completions,
ref<EvalState> evalState,
flake::LockFlags lockFlags,
Strings attrPathPrefixes,

9
src/libcmd/common-eval-args.cc
View file

@ -9,6 +9,7 @@
#include "flake/flakeref.hh"
#include "store-api.hh"
#include "command.hh"
#include "tarball.hh"
namespace nix {
@ -132,8 +133,8 @@ MixEvalArgs::MixEvalArgs()
if (to.subdir != "") extraAttrs["dir"] = to.subdir;
fetchers::overrideRegistry(from.input, to.input, extraAttrs);
}},
.completer = {[&](size_t, std::string_view prefix) {
completeFlakeRef(openStore(), prefix);
.completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
completeFlakeRef(completions, openStore(), prefix);
}}
});
@ -168,14 +169,14 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s)
{
if (EvalSettings::isPseudoUrl(s)) {
auto storePath = fetchers::downloadTarball(
state.store, EvalSettings::resolvePseudoUrl(s), "source", false).tree.storePath;
state.store, EvalSettings::resolvePseudoUrl(s), "source", false).storePath;
return state.rootPath(CanonPath(state.store->toRealPath(storePath)));
}
else if (hasPrefix(s, "flake:")) {
experimentalFeatureSettings.require(Xp::Flakes);
auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false);
auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first.storePath;
auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first;
return state.rootPath(CanonPath(state.store->toRealPath(storePath)));
}

118
src/libcmd/installables.cc
View file

@ -32,15 +32,18 @@ const static std::regex attrPathRegex(
R"((?:[a-zA-Z0-9_"-][a-zA-Z0-9_".,^\*-]*))",
std::regex::ECMAScript);
void completeFlakeInputPath(
static void completeFlakeInputPath(
AddCompletions & completions,
ref<EvalState> evalState,
const FlakeRef & flakeRef,
const std::vector<FlakeRef> & flakeRefs,
std::string_view prefix)
{
auto flake = flake::getFlake(*evalState, flakeRef, true);
for (auto & input : flake.inputs)
if (hasPrefix(input.first, prefix))
completions->add(input.first);
for (auto & flakeRef : flakeRefs) {
auto flake = flake::getFlake(*evalState, flakeRef, true);
for (auto & input : flake.inputs)
if (hasPrefix(input.first, prefix))
completions.add(input.first);
}
}
MixFlakeOptions::MixFlakeOptions()
@ -94,8 +97,8 @@ MixFlakeOptions::MixFlakeOptions()
.handler = {[&](std::string s) {
lockFlags.inputUpdates.insert(flake::parseInputPath(s));
}},
.completer = {[&](size_t, std::string_view prefix) {
needsFlakeInputCompletion = {std::string(prefix)};
.completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix);
}}
});
@ -110,11 +113,12 @@ MixFlakeOptions::MixFlakeOptions()
flake::parseInputPath(inputPath),
parseFlakeRef(flakeRef, absPath("."), true));
}},
.completer = {[&](size_t n, std::string_view prefix) {
if (n == 0)
needsFlakeInputCompletion = {std::string(prefix)};
else if (n == 1)
completeFlakeRef(getEvalState()->store, prefix);
.completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) {
if (n == 0) {
completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix);
} else if (n == 1) {
completeFlakeRef(completions, getEvalState()->store, prefix);
}
}}
});
@ -161,30 +165,12 @@ MixFlakeOptions::MixFlakeOptions()
}
}
}},
.completer = {[&](size_t, std::string_view prefix) {
completeFlakeRef(getEvalState()->store, prefix);
.completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) {
completeFlakeRef(completions, getEvalState()->store, prefix);
}}
});
}
void MixFlakeOptions::completeFlakeInput(std::string_view prefix)
{
auto evalState = getEvalState();
for (auto & flakeRefS : getFlakesForCompletion()) {
auto flakeRef = parseFlakeRefWithFragment(expandTilde(flakeRefS), absPath(".")).first;
auto flake = flake::getFlake(*evalState, flakeRef, true);
for (auto & input : flake.inputs)
if (hasPrefix(input.first, prefix))
completions->add(input.first);
}
}
void MixFlakeOptions::completionHook()
{
if (auto & prefix = needsFlakeInputCompletion)
completeFlakeInput(*prefix);
}
SourceExprCommand::SourceExprCommand()
{
addFlag({
@ -302,11 +288,18 @@ Strings SourceExprCommand::getDefaultFlakeAttrPathPrefixes()
};
}
void SourceExprCommand::completeInstallable(std::string_view prefix)
Args::CompleterClosure SourceExprCommand::getCompleteInstallable()
{
return [this](AddCompletions & completions, size_t, std::string_view prefix) {
completeInstallable(completions, prefix);
};
}
void SourceExprCommand::completeInstallable(AddCompletions & completions, std::string_view prefix)
{
try {
if (file) {
completionType = ctAttrs;
completions.setType(AddCompletions::Type::Attrs);
evalSettings.pureEval = false;
auto state = getEvalState();
@ -341,14 +334,15 @@ void SourceExprCommand::completeInstallable(std::string_view prefix)
std::string name = state->symbols[i.name];
if (name.find(searchWord) == 0) {
if (prefix_ == "")
completions->add(name);
completions.add(name);
else
completions->add(prefix_ + "." + name);
completions.add(prefix_ + "." + name);
}
}
}
} else {
completeFlakeRefWithFragment(
completions,
getEvalState(),
lockFlags,
getDefaultFlakeAttrPathPrefixes(),
@ -361,6 +355,7 @@ void SourceExprCommand::completeInstallable(std::string_view prefix)
}
void completeFlakeRefWithFragment(
AddCompletions & completions,
ref<EvalState> evalState,
flake::LockFlags lockFlags,
Strings attrPathPrefixes,
@ -373,10 +368,9 @@ void completeFlakeRefWithFragment(
bool isAttrPath = std::regex_match(prefix.begin(), prefix.end(), attrPathRegex);
auto hash = prefix.find('#');
if (!isAttrPath && hash == std::string::npos) {
completeFlakeRef(evalState->store, prefix);
completeFlakeRef(completions, evalState->store, prefix);
} else {
completionType = ctAttrs;
completions.setType(AddCompletions::Type::Attrs);
auto fragment =
isAttrPath
@ -428,9 +422,9 @@ void completeFlakeRefWithFragment(
/* Strip the attrpath prefix. */
attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size());
if (isAttrPath)
completions->add(concatStringsSep(".", evalState->symbols.resolve(attrPath2)));
completions.add(concatStringsSep(".", evalState->symbols.resolve(attrPath2)));
else
completions->add(flakeRefS + "#" + prefixRoot + concatStringsSep(".", evalState->symbols.resolve(attrPath2)));
completions.add(flakeRefS + "#" + prefixRoot + concatStringsSep(".", evalState->symbols.resolve(attrPath2)));
}
}
}
@ -441,7 +435,7 @@ void completeFlakeRefWithFragment(
for (auto & attrPath : defaultFlakeAttrPaths) {
auto attr = root->findAlongAttrPath(parseAttrPath(*evalState, attrPath));
if (!attr) continue;
completions->add(flakeRefS + "#" + prefixRoot);
completions.add(flakeRefS + "#" + prefixRoot);
}
}
}
@ -450,15 +444,15 @@ void completeFlakeRefWithFragment(
}
}
void completeFlakeRef(ref<Store> store, std::string_view prefix)
void completeFlakeRef(AddCompletions & completions, ref<Store> store, std::string_view prefix)
{
if (!experimentalFeatureSettings.isEnabled(Xp::Flakes))
return;
if (prefix == "")
completions->add(".");
completions.add(".");
completeDir(0, prefix);
Args::completeDir(completions, 0, prefix);
/* Look for registry entries that match the prefix. */
for (auto & registry : fetchers::getRegistries(store)) {
@ -467,10 +461,10 @@ void completeFlakeRef(ref<Store> store, std::string_view prefix)
if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) {
std::string from2(from, 6);
if (hasPrefix(from2, prefix))
completions->add(from2);
completions.add(from2);
} else {
if (hasPrefix(from, prefix))
completions->add(from);
completions.add(from);
}
}
}
@ -905,9 +899,7 @@ RawInstallablesCommand::RawInstallablesCommand()
expectArgs({
.label = "installables",
.handler = {&rawInstallables},
.completer = {[&](size_t, std::string_view prefix) {
completeInstallable(prefix);
}}
.completer = getCompleteInstallable(),
});
}
@ -920,6 +912,17 @@ void RawInstallablesCommand::applyDefaultInstallables(std::vector<std::string> &
}
}
std::vector<FlakeRef> RawInstallablesCommand::getFlakeRefsForCompletion()
{
applyDefaultInstallables(rawInstallables);
std::vector<FlakeRef> res;
for (auto i : rawInstallables)
res.push_back(parseFlakeRefWithFragment(
expandTilde(i),
absPath(".")).first);
return res;
}
void RawInstallablesCommand::run(ref<Store> store)
{
if (readFromStdIn && !isatty(STDIN_FILENO)) {
@ -933,10 +936,13 @@ void RawInstallablesCommand::run(ref<Store> store)
run(store, std::move(rawInstallables));
}
std::vector<std::string> RawInstallablesCommand::getFlakesForCompletion()
std::vector<FlakeRef> InstallableCommand::getFlakeRefsForCompletion()
{
applyDefaultInstallables(rawInstallables);
return rawInstallables;
return {
parseFlakeRefWithFragment(
expandTilde(_installable),
absPath(".")).first
};
}
void InstallablesCommand::run(ref<Store> store, std::vector<std::string> && rawInstallables)
@ -952,9 +958,7 @@ InstallableCommand::InstallableCommand()
.label = "installable",
.optional = true,
.handler = {&_installable},
.completer = {[&](size_t, std::string_view prefix) {
completeInstallable(prefix);
}}
.completer = getCompleteInstallable(),
});
}

2
src/libexpr/attr-path.cc
View file

@ -132,7 +132,7 @@ std::pair<SourcePath, uint32_t> findPackageFilename(EvalState & state, Value & v
if (colon == std::string::npos) fail();
std::string filename(fn, 0, colon);
auto lineno = std::stoi(std::string(fn, colon + 1, std::string::npos));
return {CanonPath(fn.substr(0, colon)), lineno};
return {SourcePath{path.accessor, CanonPath(fn.substr(0, colon))}, lineno};
} catch (std::invalid_argument & e) {
fail();
abort();

2
src/libexpr/eval-cache.cc
View file

@ -50,7 +50,7 @@ struct AttrDb
Path cacheDir = getCacheDir() + "/nix/eval-cache-v5";
createDirs(cacheDir);
Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
Path dbPath = cacheDir + "/" + fingerprint.to_string(HashFormat::Base16, false) + ".sqlite";
state->db = SQLite(dbPath);
state->db.isCache();

100
src/libexpr/eval.cc
View file

@ -12,6 +12,8 @@
#include "function-trace.hh"
#include "profiles.hh"
#include "print.hh"
#include "fs-input-accessor.hh"
#include "memory-input-accessor.hh"
#include <algorithm>
#include <chrono>
@ -503,7 +505,17 @@ EvalState::EvalState(
, sOutputSpecified(symbols.create("outputSpecified"))
, repair(NoRepair)
, emptyBindings(0)
, derivationInternal(rootPath(CanonPath("/builtin/derivation.nix")))
, rootFS(makeFSInputAccessor(CanonPath::root))
, corepkgsFS(makeMemoryInputAccessor())
, internalFS(makeMemoryInputAccessor())
, derivationInternal{corepkgsFS->addFile(
CanonPath("derivation-internal.nix"),
#include "primops/derivation.nix.gen.hh"
)}
, callFlakeInternal{internalFS->addFile(
CanonPath("call-flake.nix"),
#include "flake/call-flake.nix.gen.hh"
)}
, store(store)
, buildStore(buildStore ? buildStore : store)
, debugRepl(nullptr)
@ -539,7 +551,7 @@ EvalState::EvalState(
auto r = resolveSearchPathPath(i.path);
if (!r) continue;
auto path = *std::move(r);
auto path = std::move(*r);
if (store->isInStore(path)) {
try {
@ -555,6 +567,11 @@ EvalState::EvalState(
}
}
corepkgsFS->addFile(
CanonPath("fetchurl.nix"),
#include "fetchurl.nix.gen.hh"
);
createBaseEnv();
}
@ -585,6 +602,9 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value &
SourcePath EvalState::checkSourcePath(const SourcePath & path_)
{
// Don't check non-rootFS accessors, they're in a different namespace.
if (path_.accessor != ref<InputAccessor>(rootFS)) return path_;
if (!allowedPaths) return path_;
auto i = resolvedPaths.find(path_.path.abs());
@ -599,8 +619,6 @@ SourcePath EvalState::checkSourcePath(const SourcePath & path_)
*/
Path abspath = canonPath(path_.path.abs());
if (hasPrefix(abspath, corepkgsPrefix)) return CanonPath(abspath);
for (auto & i : *allowedPaths) {
if (isDirOrInDir(abspath, i)) {
found = true;
@ -617,7 +635,7 @@ SourcePath EvalState::checkSourcePath(const SourcePath & path_)
/* Resolve symlinks. */
debug("checking access to '%s'", abspath);
SourcePath path = CanonPath(canonPath(abspath, true));
SourcePath path = rootPath(CanonPath(canonPath(abspath, true)));
for (auto & i : *allowedPaths) {
if (isDirOrInDir(path.path.abs(), i)) {
@ -649,12 +667,12 @@ void EvalState::checkURI(const std::string & uri)
/* If the URI is a path, then check it against allowedPaths as
well. */
if (hasPrefix(uri, "/")) {
checkSourcePath(CanonPath(uri));
checkSourcePath(rootPath(CanonPath(uri)));
return;
}
if (hasPrefix(uri, "file://")) {
checkSourcePath(CanonPath(std::string(uri, 7)));
checkSourcePath(rootPath(CanonPath(std::string(uri, 7))));
return;
}
@ -950,7 +968,7 @@ void Value::mkStringMove(const char * s, const NixStringContext & context)
void Value::mkPath(const SourcePath & path)
{
mkPath(makeImmutableString(path.path.abs()));
mkPath(&*path.accessor, makeImmutableString(path.path.abs()));
}
@ -1035,7 +1053,7 @@ std::string EvalState::mkOutputStringRaw(
/* In practice, this is testing for the case of CA derivations, or
dynamic derivations. */
return optStaticOutputPath
? store->printStorePath(*std::move(optStaticOutputPath))
? store->printStorePath(std::move(*optStaticOutputPath))
/* Downstream we would substitute this for an actual path once
we build the floating CA derivation */
: DownstreamPlaceholder::fromSingleDerivedPathBuilt(b, xpSettings).render();
@ -1165,24 +1183,6 @@ void EvalState::evalFile(const SourcePath & path_, Value & v, bool mustBeTrivial
if (!e)
e = parseExprFromFile(checkSourcePath(resolvedPath));
cacheFile(path, resolvedPath, e, v, mustBeTrivial);
}
void EvalState::resetFileCache()
{
fileEvalCache.clear();
fileParseCache.clear();
}
void EvalState::cacheFile(
const SourcePath & path,
const SourcePath & resolvedPath,
Expr * e,
Value & v,
bool mustBeTrivial)
{
fileParseCache[resolvedPath] = e;
try {
@ -1211,6 +1211,13 @@ void EvalState::cacheFile(
}
void EvalState::resetFileCache()
{
fileEvalCache.clear();
fileParseCache.clear();
}
void EvalState::eval(Expr * e, Value & v)
{
e->eval(*this, baseEnv, v);
@ -2037,7 +2044,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
else if (firstType == nPath) {
if (!context.empty())
state.error("a string that refers to a store path cannot be appended to a path").atPos(pos).withFrame(env, *this).debugThrow<EvalError>();
v.mkPath(CanonPath(canonPath(str())));
v.mkPath(state.rootPath(CanonPath(canonPath(str()))));
} else
v.mkStringMove(c_str(), context);
}
@ -2236,7 +2243,7 @@ BackedStringView EvalState::coerceToString(
!canonicalizePath && !copyToStore
? // FIXME: hack to preserve path literals that end in a
// slash, as in /foo/${x}.
v._path
v._path.path
: copyToStore
? store->printStorePath(copyPathToStore(context, v.path()))
: std::string(v.path().path.abs());
@ -2290,7 +2297,7 @@ BackedStringView EvalState::coerceToString(
&& (!v2->isList() || v2->listSize() != 0))
result += " ";
}
return std::move(result);
return result;
}
}
@ -2310,7 +2317,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
auto dstPath = i != srcToStore.end()
? i->second
: [&]() {
auto dstPath = path.fetchToStore(store, path.baseName(), nullptr, repair);
auto dstPath = path.fetchToStore(store, path.baseName(), FileIngestionMethod::Recursive, nullptr, repair);
allowPath(dstPath);
srcToStore.insert_or_assign(path, dstPath);
printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath));
@ -2326,10 +2333,34 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat
SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx)
{
try {
forceValue(v, pos);
} catch (Error & e) {
e.addTrace(positions[pos], errorCtx);
throw;
}
/* Handle path values directly, without coercing to a string. */
if (v.type() == nPath)
return v.path();
/* Similarly, handle __toString where the result may be a path
value. */
if (v.type() == nAttrs) {
auto i = v.attrs->find(sToString);
if (i != v.attrs->end()) {
Value v1;
callFunction(*i->value, v, v1, pos);
return coerceToPath(pos, v1, context, errorCtx);
}
}
/* Any other value should be coercable to a string, interpreted
relative to the root filesystem. */
auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned();
if (path == "" || path[0] != '/')
error("string '%1%' doesn't represent an absolute path", path).withTrace(pos, errorCtx).debugThrow<EvalError>();
return CanonPath(path);
return rootPath(CanonPath(path));
}
@ -2429,7 +2460,10 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v
return v1.string_view().compare(v2.string_view()) == 0;
case nPath:
return strcmp(v1._path, v2._path) == 0;
return
// FIXME: compare accessors by their fingerprint.
v1._path.accessor == v2._path.accessor
&& strcmp(v1._path.path, v2._path.path) == 0;
case nNull:
return true;

35
src/libexpr/eval.hh
View file

@ -24,6 +24,8 @@ class EvalState;
class StorePath;
struct SingleDerivedPath;
enum RepairFlag : bool;
struct FSInputAccessor;
struct MemoryInputAccessor;
/**
@ -211,8 +213,26 @@ public:
Bindings emptyBindings;
/**
* The accessor for the root filesystem.
*/
const ref<FSInputAccessor> rootFS;
/**
* The in-memory filesystem for <nix/...> paths.
*/
const ref<MemoryInputAccessor> corepkgsFS;
/**
* In-memory filesystem for internal, non-user-callable Nix
* expressions like call-flake.nix.
*/
const ref<MemoryInputAccessor> internalFS;
const SourcePath derivationInternal;
const SourcePath callFlakeInternal;
/**
* Store used to materialise .drv files.
*/
@ -223,7 +243,6 @@ public:
*/
const ref<Store> buildStore;
RootValue vCallFlake = nullptr;
RootValue vImportedDrvToDerivation = nullptr;
/**
@ -405,16 +424,6 @@ public:
*/
void evalFile(const SourcePath & path, Value & v, bool mustBeTrivial = false);
/**
* Like `evalFile`, but with an already parsed expression.
*/
void cacheFile(
const SourcePath & path,
const SourcePath & resolvedPath,
Expr * e,
Value & v,
bool mustBeTrivial = false);
void resetFileCache();
/**
@ -424,7 +433,7 @@ public:
SourcePath findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos);
/**
* Try to resolve a search path value (not the optinal key part)
* Try to resolve a search path value (not the optional key part)
*
* If the specified search path element is a URI, download it.
*
@ -829,8 +838,6 @@ struct InvalidPathError : EvalError
#endif
};
static const std::string corepkgsPrefix{"/__corepkgs__/"};
template<class ErrorType>
void ErrorBuilder::debugThrow()
{

48
src/libexpr/flake/flake.cc
View file

@ -15,7 +15,7 @@ using namespace flake;
namespace flake {
typedef std::pair<fetchers::Tree, FlakeRef> FetchedFlake;
typedef std::pair<StorePath, FlakeRef> FetchedFlake;
typedef std::vector<std::pair<FlakeRef, FetchedFlake>> FlakeCache;
static std::optional<FetchedFlake> lookupInFlakeCache(
@ -34,7 +34,7 @@ static std::optional<FetchedFlake> lookupInFlakeCache(
return std::nullopt;
}
static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree(
static std::tuple<StorePath, FlakeRef, FlakeRef> fetchOrSubstituteTree(
EvalState & state,
const FlakeRef & originalRef,
bool allowLookup,
@ -61,16 +61,16 @@ static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree(
flakeCache.push_back({originalRef, *fetched});
}
auto [tree, lockedRef] = *fetched;
auto [storePath, lockedRef] = *fetched;
debug("got tree '%s' from '%s'",
state.store->printStorePath(tree.storePath), lockedRef);
state.store->printStorePath(storePath), lockedRef);
state.allowPath(tree.storePath);
state.allowPath(storePath);
assert(!originalRef.input.getNarHash() || tree.storePath == originalRef.input.computeStorePath(*state.store));
assert(!originalRef.input.getNarHash() || storePath == originalRef.input.computeStorePath(*state.store));
return {std::move(tree), resolvedRef, lockedRef};
return {std::move(storePath), resolvedRef, lockedRef};
}
static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos)
@ -202,30 +202,30 @@ static Flake getFlake(
FlakeCache & flakeCache,
InputPath lockRootPath)
{
auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree(
auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree(
state, originalRef, allowLookup, flakeCache);
// Guard against symlink attacks.
auto flakeDir = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir, true);
auto flakeDir = canonPath(state.store->toRealPath(storePath) + "/" + lockedRef.subdir, true);
auto flakeFile = canonPath(flakeDir + "/flake.nix", true);
if (!isInDir(flakeFile, sourceInfo.actualPath))
if (!isInDir(flakeFile, state.store->toRealPath(storePath)))
throw Error("'flake.nix' file of flake '%s' escapes from '%s'",
lockedRef, state.store->printStorePath(sourceInfo.storePath));
lockedRef, state.store->printStorePath(storePath));
Flake flake {
.originalRef = originalRef,
.resolvedRef = resolvedRef,
.lockedRef = lockedRef,
.sourceInfo = std::make_shared<fetchers::Tree>(std::move(sourceInfo))
.storePath = storePath,
};
if (!pathExists(flakeFile))
throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir);
Value vInfo;
state.evalFile(CanonPath(flakeFile), vInfo, false); // FIXME: symlink attack
state.evalFile(state.rootPath(CanonPath(flakeFile)), vInfo, false); // FIXME: symlink attack
expectType(state, nAttrs, vInfo, state.positions.add({CanonPath(flakeFile)}, 1, 1));
expectType(state, nAttrs, vInfo, state.positions.add({state.rootPath(CanonPath(flakeFile))}, 1, 1));
if (auto description = vInfo.attrs->get(state.sDescription)) {
expectType(state, nString, *description->value, description->pos);
@ -346,7 +346,7 @@ LockedFlake lockFlake(
// FIXME: symlink attack
auto oldLockFile = LockFile::read(
lockFlags.referenceLockFilePath.value_or(
flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock"));
state.store->toRealPath(flake.storePath) + "/" + flake.lockedRef.subdir + "/flake.lock"));
debug("old lock file: %s", oldLockFile);
@ -574,7 +574,7 @@ LockedFlake lockFlake(
oldLock
? std::dynamic_pointer_cast<const Node>(oldLock)
: LockFile::read(
inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root.get_ptr(),
state.store->toRealPath(inputFlake.storePath) + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root.get_ptr(),
oldLock ? lockRootPath : inputPath,
localPath,
false);
@ -598,7 +598,7 @@ LockedFlake lockFlake(
};
// Bring in the current ref for relative path resolution if we have it
auto parentPath = canonPath(flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir, true);
auto parentPath = canonPath(state.store->toRealPath(flake.storePath) + "/" + flake.lockedRef.subdir, true);
computeLocks(
flake.inputs,
@ -729,7 +729,7 @@ void callFlake(EvalState & state,
emitTreeAttrs(
state,
*lockedFlake.flake.sourceInfo,
lockedFlake.flake.storePath,
lockedFlake.flake.lockedRef.input,
*vRootSrc,
false,
@ -737,14 +737,10 @@ void callFlake(EvalState & state,
vRootSubdir->mkString(lockedFlake.flake.lockedRef.subdir);
if (!state.vCallFlake) {
state.vCallFlake = allocRootValue(state.allocValue());
state.eval(state.parseExprFromString(
#include "call-flake.nix.gen.hh"
, CanonPath::root), **state.vCallFlake);
}
auto vCallFlake = state.allocValue();
state.evalFile(state.callFlakeInternal, *vCallFlake);
state.callFunction(**state.vCallFlake, *vLocks, *vTmp1, noPos);
state.callFunction(*vCallFlake, *vLocks, *vTmp1, noPos);
state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos);
state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos);
}
@ -893,7 +889,7 @@ Fingerprint LockedFlake::getFingerprint() const
// flake.sourceInfo.storePath for the fingerprint.
return hashString(htSHA256,
fmt("%s;%s;%d;%d;%s",
flake.sourceInfo->storePath.to_string(),
flake.storePath.to_string(),
flake.lockedRef.subdir,
flake.lockedRef.input.getRevCount().value_or(0),
flake.lockedRef.input.getLastModified().value_or(0),

6
src/libexpr/flake/flake.hh
View file

@ -10,8 +10,6 @@ namespace nix {
class EvalState;
namespace fetchers { struct Tree; }
namespace flake {
struct FlakeInput;
@ -84,7 +82,7 @@ struct Flake
*/
bool forceDirty = false;
std::optional<std::string> description;
std::shared_ptr<const fetchers::Tree> sourceInfo;
StorePath storePath;
FlakeInputs inputs;
/**
* 'nixConfig' attribute
@ -193,7 +191,7 @@ void callFlake(
void emitTreeAttrs(
EvalState & state,
const fetchers::Tree & tree,
const StorePath & storePath,
const fetchers::Input & input,
Value & v,
bool emptyRevFallback = false,

6
src/libexpr/flake/flakeref.cc
View file

@ -272,10 +272,10 @@ FlakeRef FlakeRef::fromAttrs(const fetchers::Attrs & attrs)
fetchers::maybeGetStrAttr(attrs, "dir").value_or(""));
}
std::pair<fetchers::Tree, FlakeRef> FlakeRef::fetchTree(ref<Store> store) const
std::pair<StorePath, FlakeRef> FlakeRef::fetchTree(ref<Store> store) const
{
auto [tree, lockedInput] = input.fetch(store);
return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)};
auto [storePath, lockedInput] = input.fetch(store);
return {std::move(storePath), FlakeRef(std::move(lockedInput), subdir)};
}
std::tuple<FlakeRef, std::string, ExtendedOutputsSpec> parseFlakeRefWithFragmentAndExtendedOutputsSpec(

2
src/libexpr/flake/flakeref.hh
View file

@ -63,7 +63,7 @@ struct FlakeRef
static FlakeRef fromAttrs(const fetchers::Attrs & attrs);
std::pair<fetchers::Tree, FlakeRef> fetchTree(ref<Store> store) const;
std::pair<StorePath, FlakeRef> fetchTree(ref<Store> store) const;
};
std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef);

24
src/libexpr/flake/lockfile.cc
View file

@ -2,8 +2,10 @@
#include "store-api.hh"
#include "url-parts.hh"
#include <algorithm>
#include <iomanip>
#include <iterator>
#include <nlohmann/json.hpp>
namespace nix::flake {
@ -45,16 +47,26 @@ StorePath LockedNode::computeStorePath(Store & store) const
return lockedRef.input.computeStorePath(store);
}
std::shared_ptr<Node> LockFile::findInput(const InputPath & path)
{
static std::shared_ptr<Node> doFind(const ref<Node>& root, const InputPath & path, std::vector<InputPath>& visited) {
auto pos = root;
auto found = std::find(visited.cbegin(), visited.cend(), path);
if(found != visited.end()) {
std::vector<std::string> cycle;
std::transform(found, visited.cend(), std::back_inserter(cycle), printInputPath);
cycle.push_back(printInputPath(path));
throw Error("follow cycle detected: [%s]", concatStringsSep(" -> ", cycle));
}
visited.push_back(path);
for (auto & elem : path) {
if (auto i = get(pos->inputs, elem)) {
if (auto node = std::get_if<0>(&*i))
pos = *node;
else if (auto follows = std::get_if<1>(&*i)) {
if (auto p = findInput(*follows))
if (auto p = doFind(root, *follows, visited))
pos = ref(p);
else
return {};
@ -66,6 +78,12 @@ std::shared_ptr<Node> LockFile::findInput(const InputPath & path)
return pos;
}
std::shared_ptr<Node> LockFile::findInput(const InputPath & path)
{
std::vector<InputPath> visited;
return doFind(root, path, visited);
}
LockFile::LockFile(const nlohmann::json & json, const Path & path)
{
auto version = json.value("version", 0);

7
src/libexpr/nixexpr.hh
View file

@ -20,7 +20,6 @@ MakeError(Abort, EvalError);
MakeError(TypeError, EvalError);
MakeError(UndefinedVarError, Error);
MakeError(MissingArgumentError, EvalError);
MakeError(RestrictedPathError, Error);
/**
* Position objects.
@ -200,9 +199,13 @@ struct ExprString : Expr
struct ExprPath : Expr
{
ref<InputAccessor> accessor;
std::string s;
Value v;
ExprPath(std::string s) : s(std::move(s)) { v.mkPath(this->s.c_str()); };
ExprPath(ref<InputAccessor> accessor, std::string s) : accessor(accessor), s(std::move(s))
{
v.mkPath(&*accessor, this->s.c_str());
}
Value * maybeThunk(EvalState & state, Env & env) override;
COMMON_METHODS
};

16
src/libexpr/parser.y
View file

@ -522,7 +522,7 @@ path_start
/* add back in the trailing '/' to the first segment */
if ($1.p[$1.l-1] == '/' && $1.l > 1)
path += "/";
$$ = new ExprPath(std::move(path));
$$ = new ExprPath(ref<InputAccessor>(data->state.rootFS), std::move(path));
}
| HPATH {
if (evalSettings.pureEval) {
@ -532,7 +532,7 @@ path_start
);
}
Path path(getHome() + std::string($1.p + 1, $1.l - 1));
$$ = new ExprPath(std::move(path));
$$ = new ExprPath(ref<InputAccessor>(data->state.rootFS), std::move(path));
}
;
@ -648,9 +648,11 @@ formal
#include "eval.hh"
#include "filetransfer.hh"
#include "fetchers.hh"
#include "tarball.hh"
#include "store-api.hh"
#include "flake/flake.hh"
#include "fs-input-accessor.hh"
#include "memory-input-accessor.hh"
namespace nix {
@ -758,11 +760,11 @@ SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_
auto r = *rOpt;
Path res = suffix == "" ? r : concatStrings(r, "/", suffix);
if (pathExists(res)) return CanonPath(canonPath(res));
if (pathExists(res)) return rootPath(CanonPath(canonPath(res)));
}
if (hasPrefix(path, "nix/"))
return CanonPath(concatStrings(corepkgsPrefix, path.substr(4)));
return {corepkgsFS, CanonPath(path.substr(3))};
debugThrow(ThrownError({
.msg = hintfmt(evalSettings.pureEval
@ -785,7 +787,7 @@ std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Pa
if (EvalSettings::isPseudoUrl(value)) {
try {
auto storePath = fetchers::downloadTarball(
store, EvalSettings::resolvePseudoUrl(value), "source", false).tree.storePath;
store, EvalSettings::resolvePseudoUrl(value), "source", false).storePath;
res = { store->toRealPath(storePath) };
} catch (FileTransferError & e) {
logWarning({
@ -799,7 +801,7 @@ std::optional<std::string> EvalState::resolveSearchPathPath(const SearchPath::Pa
experimentalFeatureSettings.require(Xp::Flakes);
auto flakeRef = parseFlakeRef(value.substr(6), {}, true, false);
debug("fetching flake search path element '%s''", value);
auto storePath = flakeRef.resolve(store).fetchTree(store).first.storePath;
auto storePath = flakeRef.resolve(store).fetchTree(store).first;
res = { store->toRealPath(storePath) };
}

3
src/libexpr/paths.cc
View file

@ -1,10 +1,11 @@
#include "eval.hh"
#include "fs-input-accessor.hh"
namespace nix {
SourcePath EvalState::rootPath(CanonPath path)
{
return std::move(path);
return {rootFS, std::move(path)};
}
}

158
src/libexpr/primops.cc
View file

@ -121,13 +121,15 @@ static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, co
auto path = state.coerceToPath(noPos, v, context, "while realising the context of a path");
try {
StringMap rewrites = state.realiseContext(context);
auto realPath = state.rootPath(CanonPath(state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context)));
if (!context.empty()) {
auto rewrites = state.realiseContext(context);
auto realPath = state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context);
return {path.accessor, CanonPath(realPath)};
}
return flags.checkForPureEval
? state.checkSourcePath(realPath)
: realPath;
? state.checkSourcePath(path)
: path;
} catch (Error & e) {
e.addTrace(state.positions[pos], "while realising the context of path '%s'", path);
throw;
@ -202,7 +204,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v
state.vImportedDrvToDerivation = allocRootValue(state.allocValue());
state.eval(state.parseExprFromString(
#include "imported-drv-to-derivation.nix.gen.hh"
, CanonPath::root), **state.vImportedDrvToDerivation);
, state.rootPath(CanonPath::root)), **state.vImportedDrvToDerivation);
}
state.forceFunction(**state.vImportedDrvToDerivation, pos, "while evaluating imported-drv-to-derivation.nix.gen.hh");
@ -210,12 +212,6 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v
state.forceAttrs(v, pos, "while calling imported-drv-to-derivation.nix.gen.hh");
}
else if (path2 == corepkgsPrefix + "fetchurl.nix") {
state.eval(state.parseExprFromString(
#include "fetchurl.nix.gen.hh"
, CanonPath::root), v);
}
else {
if (!vScope)
state.evalFile(path, v);
@ -599,7 +595,10 @@ struct CompareValues
case nString:
return v1->string_view().compare(v2->string_view()) < 0;
case nPath:
return strcmp(v1->_path, v2->_path) < 0;
// Note: we don't take the accessor into account
// since it's not obvious how to compare them in a
// reproducible way.
return strcmp(v1->_path.path, v2->_path.path) < 0;
case nList:
// Lexicographic comparison
for (size_t i = 0;; i++) {
@ -734,6 +733,14 @@ static RegisterPrimOp primop_genericClosure(PrimOp {
```
[ { key = 5; } { key = 16; } { key = 8; } { key = 4; } { key = 2; } { key = 1; } ]
```
`key` can be one of the following types:
- [Number](@docroot@/language/values.md#type-number)
- [Boolean](@docroot@/language/values.md#type-boolean)
- [String](@docroot@/language/values.md#type-string)
- [Path](@docroot@/language/values.md#type-path)
- [List](@docroot@/language/values.md#list)
)",
.fun = prim_genericClosure,
});
@ -1485,7 +1492,7 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args,
}));
NixStringContext context;
auto path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.storePath")).path;
auto path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'")).path;
/* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink
directly in the store. The latter condition is necessary so
e.g. nix-push does the right thing. */
@ -1758,7 +1765,7 @@ static void prim_hashFile(EvalState & state, const PosIdx pos, Value * * args, V
auto path = realisePath(state, pos, *args[1]);
v.mkString(hashString(*ht, path.readFile()).to_string(Base16, false));
v.mkString(hashString(*ht, path.readFile()).to_string(HashFormat::Base16, false));
}
static RegisterPrimOp primop_hashFile({
@ -2203,7 +2210,7 @@ static void addPath(
path = evalSettings.pureEval && expectedHash
? path
: state.checkSourcePath(CanonPath(path)).path.abs();
: state.checkSourcePath(state.rootPath(CanonPath(path))).path.abs();
PathFilter filter = filterFun ? ([&](const Path & path) {
auto st = lstat(path);
@ -2236,9 +2243,7 @@ static void addPath(
});
if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
StorePath dstPath = settings.readOnlyMode
? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first
: state.store->addToStore(name, path, method, htSHA256, filter, state.repair, refs);
auto dstPath = state.rootPath(CanonPath(path)).fetchToStore(state.store, name, method, &filter, state.repair);
if (expectedHash && expectedStorePath != dstPath)
state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path));
state.allowAndSetStorePathString(dstPath, v);
@ -2255,7 +2260,7 @@ static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * arg
{
NixStringContext context;
auto path = state.coerceToPath(pos, *args[1], context,
"while evaluating the second argument (the path to filter) passed to builtins.filterSource");
"while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'");
state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource");
addPath(state, pos, path.baseName(), path.path.abs(), args[0], FileIngestionMethod::Recursive, std::nullopt, v, context);
}
@ -3715,10 +3720,11 @@ static RegisterPrimOp primop_substring({
.doc = R"(
Return the substring of *s* from character position *start*
(zero-based) up to but not including *start + len*. If *start* is
greater than the length of the string, an empty string is returned,
and if *start + len* lies beyond the end of the string, only the
substring up to the end of the string is returned. *start* must be
non-negative. For example,
greater than the length of the string, an empty string is returned.
If *start + len* lies beyond the end of the string or *len* is `-1`,
only the substring up to the end of the string is returned.
*start* must be non-negative.
For example,
```nix
builtins.substring 0 3 "nixos"
@ -3760,7 +3766,7 @@ static void prim_hashString(EvalState & state, const PosIdx pos, Value * * args,
NixStringContext context; // discarded
auto s = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString");
v.mkString(hashString(*ht, s).to_string(Base16, false));
v.mkString(hashString(*ht, s).to_string(HashFormat::Base16, false));
}
static RegisterPrimOp primop_hashString({
@ -3774,6 +3780,101 @@ static RegisterPrimOp primop_hashString({
.fun = prim_hashString,
});
static void prim_convertHash(EvalState & state, const PosIdx pos, Value * * args, Value & v)
{
state.forceAttrs(*args[0], pos, "while evaluating the first argument passed to builtins.convertHash");
auto &inputAttrs = args[0]->attrs;
Bindings::iterator iteratorHash = getAttr(state, state.symbols.create("hash"), inputAttrs, "while locating the attribute 'hash'");
auto hash = state.forceStringNoCtx(*iteratorHash->value, pos, "while evaluating the attribute 'hash'");
Bindings::iterator iteratorHashAlgo = inputAttrs->find(state.symbols.create("hashAlgo"));
std::optional<HashType> ht = std::nullopt;
if (iteratorHashAlgo != inputAttrs->end()) {
ht = parseHashType(state.forceStringNoCtx(*iteratorHashAlgo->value, pos, "while evaluating the attribute 'hashAlgo'"));
}
Bindings::iterator iteratorToHashFormat = getAttr(state, state.symbols.create("toHashFormat"), args[0]->attrs, "while locating the attribute 'toHashFormat'");
HashFormat hf = parseHashFormat(state.forceStringNoCtx(*iteratorToHashFormat->value, pos, "while evaluating the attribute 'toHashFormat'"));
v.mkString(Hash::parseAny(hash, ht).to_string(hf, hf == HashFormat::SRI));
}
static RegisterPrimOp primop_convertHash({
.name = "__convertHash",
.args = {"args"},
.doc = R"(
Return the specified representation of a hash string, based on the attributes presented in *args*:
- `hash`
The hash to be converted.
The hash format is detected automatically.
- `hashAlgo`
The algorithm used to create the hash. Must be one of
- `"md5"`
- `"sha1"`
- `"sha256"`
- `"sha512"`
The attribute may be omitted when `hash` is an [SRI hash](https://www.w3.org/TR/SRI/#the-integrity-attribute) or when the hash is prefixed with the hash algorithm name followed by a colon.
That `<hashAlgo>:<hashBody>` syntax is supported for backwards compatibility with existing tooling.
- `toHashFormat`
The format of the resulting hash. Must be one of
- `"base16"`
- `"base32"`
- `"base64"`
- `"sri"`
The result hash is the *toHashFormat* representation of the hash *hash*.
> **Example**
>
> Convert a SHA256 hash in Base16 to SRI:
>
> ```nix
> builtins.convertHash {
> hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
> toHashFormat = "sri";
> hashAlgo = "sha256";
> }
> ```
>
> "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="
> **Example**
>
> Convert a SHA256 hash in SRI to Base16:
>
> ```nix
> builtins.convertHash {
> hash = "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=";
> toHashFormat = "base16";
> }
> ```
>
> "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
> **Example**
>
> Convert a hash in the form `<hashAlgo>:<hashBody>` in Base16 to SRI:
>
> ```nix
> builtins.convertHash {
> hash = "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
> toHashFormat = "sri";
> }
> ```
>
> "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="
)",
.fun = prim_convertHash,
});
struct RegexCache
{
// TODO use C++20 transparent comparison when available
@ -4450,12 +4551,7 @@ void EvalState::createBaseEnv()
/* Note: we have to initialize the 'derivation' constant *after*
building baseEnv/staticBaseEnv because it uses 'builtins'. */
char code[] =
#include "primops/derivation.nix.gen.hh"
// the parser needs two NUL bytes as terminators; one of them
// is implied by being a C string.
"\0";
eval(parse(code, sizeof(code), derivationInternal, {CanonPath::root}, staticBaseEnv), *vDerivation);
evalFile(derivationInternal, *vDerivation);
}

97
src/libexpr/primops/context.cc
View file

@ -30,20 +30,27 @@ static RegisterPrimOp primop_hasContext({
.name = "__hasContext",
.args = {"s"},
.doc = R"(
Return `true` if string *s* has a non-empty context. The
context can be obtained with
Return `true` if string *s* has a non-empty context.
The context can be obtained with
[`getContext`](#builtins-getContext).
> **Example**
>
> Many operations require a string context to be empty because they are intended only to work with "regular" strings, and also to help users avoid unintentionally loosing track of string context elements.
> `builtins.hasContext` can help create better domain-specific errors in those case.
>
> ```nix
> name: meta:
>
> if builtins.hasContext name
> then throw "package name cannot contain string context"
> else { ${name} = meta; }
> ```
)",
.fun = prim_hasContext
});
/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a
builder without causing the derivation to be built (for instance,
in the derivation that builds NARs in nix-push, when doing
source-only deployment). This primop marks the string context so
that builtins.derivation adds the path to drv.inputSrcs rather than
drv.inputDrvs. */
static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx pos, Value * * args, Value & v)
{
NixStringContext context;
@ -66,11 +73,83 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx p
static RegisterPrimOp primop_unsafeDiscardOutputDependency({
.name = "__unsafeDiscardOutputDependency",
.arity = 1,
.args = {"s"},
.doc = R"(
Create a copy of the given string where every "derivation deep" string context element is turned into a constant string context element.
This is the opposite of [`builtins.addDrvOutputDependencies`](#builtins-addDrvOutputDependencies).
This is unsafe because it allows us to "forget" store objects we would have otherwise refered to with the string context,
whereas Nix normally tracks all dependencies consistently.
Safe operations "grow" but never "shrink" string contexts.
[`builtins.addDrvOutputDependencies`] in contrast is safe because "derivation deep" string context element always refers to the underlying derivation (among many more things).
Replacing a constant string context element with a "derivation deep" element is a safe operation that just enlargens the string context without forgetting anything.
[`builtins.addDrvOutputDependencies`]: #builtins-addDrvOutputDependencies
)",
.fun = prim_unsafeDiscardOutputDependency
});
static void prim_addDrvOutputDependencies(EvalState & state, const PosIdx pos, Value * * args, Value & v)
{
NixStringContext context;
auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.addDrvOutputDependencies");
auto contextSize = context.size();
if (contextSize != 1) {
throw EvalError({
.msg = hintfmt("context of string '%s' must have exactly one element, but has %d", *s, contextSize),
.errPos = state.positions[pos]
});
}
NixStringContext context2 {
(NixStringContextElem { std::visit(overloaded {
[&](const NixStringContextElem::Opaque & c) -> NixStringContextElem::DrvDeep {
if (!c.path.isDerivation()) {
throw EvalError({
.msg = hintfmt("path '%s' is not a derivation",
state.store->printStorePath(c.path)),
.errPos = state.positions[pos],
});
}
return NixStringContextElem::DrvDeep {
.drvPath = c.path,
};
},
[&](const NixStringContextElem::Built & c) -> NixStringContextElem::DrvDeep {
throw EvalError({
.msg = hintfmt("`addDrvOutputDependencies` can only act on derivations, not on a derivation output such as '%1%'", c.output),
.errPos = state.positions[pos],
});
},
[&](const NixStringContextElem::DrvDeep & c) -> NixStringContextElem::DrvDeep {
/* Reuse original item because we want this to be idempotent. */
return std::move(c);
},
}, context.begin()->raw) }),
};
v.mkString(*s, context2);
}
static RegisterPrimOp primop_addDrvOutputDependencies({
.name = "__addDrvOutputDependencies",
.args = {"s"},
.doc = R"(
Create a copy of the given string where a single consant string context element is turned into a "derivation deep" string context element.
The store path that is the constant string context element should point to a valid derivation, and end in `.drv`.
The original string context element must not be empty or have multiple elements, and it must not have any other type of element other than a constant or derivation deep element.
The latter is supported so this function is idempotent.
This is the opposite of [`builtins.unsafeDiscardOutputDependency`](#builtins-addDrvOutputDependencies).
)",
.fun = prim_addDrvOutputDependencies
});
/* Extract the context of a string as a structured Nix value.
The context is represented as an attribute set whose keys are the

6
src/libexpr/primops/fetchMercurial.cc
View file

@ -71,10 +71,10 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a
auto input = fetchers::Input::fromAttrs(std::move(attrs));
// FIXME: use name
auto [tree, input2] = input.fetch(state.store);
auto [storePath, input2] = input.fetch(state.store);
auto attrs2 = state.buildBindings(8);
state.mkStorePathString(tree.storePath, attrs2.alloc(state.sOutPath));
state.mkStorePathString(storePath, attrs2.alloc(state.sOutPath));
if (input2.getRef())
attrs2.alloc("branch").mkString(*input2.getRef());
// Backward compatibility: set 'rev' to
@ -86,7 +86,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a
attrs2.alloc("revCount").mkInt(*revCount);
v.mkAttrs(attrs2);
state.allowPath(tree.storePath);
state.allowPath(storePath);
}
static RegisterPrimOp r_fetchMercurial({

29
src/libexpr/primops/fetchTree.cc
View file

@ -5,6 +5,7 @@
#include "fetchers.hh"
#include "filetransfer.hh"
#include "registry.hh"
#include "tarball.hh"
#include "url.hh"
#include <ctime>
@ -15,7 +16,7 @@ namespace nix {
void emitTreeAttrs(
EvalState & state,
const fetchers::Tree & tree,
const StorePath & storePath,
const fetchers::Input & input,
Value & v,
bool emptyRevFallback,
@ -25,13 +26,13 @@ void emitTreeAttrs(
auto attrs = state.buildBindings(10);
state.mkStorePathString(tree.storePath, attrs.alloc(state.sOutPath));
state.mkStorePathString(storePath, attrs.alloc(state.sOutPath));
// FIXME: support arbitrary input attributes.
auto narHash = input.getNarHash();
assert(narHash);
attrs.alloc("narHash").mkString(narHash->to_string(SRI, true));
attrs.alloc("narHash").mkString(narHash->to_string(HashFormat::SRI, true));
if (input.getType() == "git")
attrs.alloc("submodules").mkBool(
@ -148,6 +149,11 @@ static void fetchTree(
attrs.emplace("url", fixGitURL(url));
input = fetchers::Input::fromAttrs(std::move(attrs));
} else {
if (!experimentalFeatureSettings.isEnabled(Xp::Flakes))
state.debugThrowLastTrace(EvalError({
.msg = hintfmt("passing a string argument to 'fetchTree' requires the 'flakes' experimental feature"),
.errPos = state.positions[pos]
}));
input = fetchers::Input::fromURL(url);
}
}
@ -160,11 +166,11 @@ static void fetchTree(
state.checkURI(input.toURLString());
auto [tree, input2] = input.fetch(state.store);
auto [storePath, input2] = input.fetch(state.store);
state.allowPath(tree.storePath);
state.allowPath(storePath);
emitTreeAttrs(state, tree, input2, v, params.emptyRevFallback, false);
emitTreeAttrs(state, storePath, input2, v, params.emptyRevFallback, false);
}
static void prim_fetchTree(EvalState & state, const PosIdx pos, Value * * args, Value & v)
@ -180,6 +186,10 @@ static RegisterPrimOp primop_fetchTree({
*input* must be a [flake reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references), either in attribute set representation or in the URL-like syntax.
The input should be "locked", that is, it should contain a commit hash or content hash unless impure evaluation (`--impure`) is enabled.
> **Note**
>
> The URL-like syntax requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled.
Here are some examples of how to use `fetchTree`:
- Fetch a GitHub repository using the attribute set representation:
@ -213,7 +223,6 @@ static RegisterPrimOp primop_fetchTree({
```
)",
.fun = prim_fetchTree,
.experimentalFeature = Xp::Flakes,
});
static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v,
@ -280,7 +289,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
// https://github.com/NixOS/nix/issues/4313
auto storePath =
unpack
? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).tree.storePath
? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath
: fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath;
if (expectedHash) {
@ -289,7 +298,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
: hashFile(htSHA256, state.store->toRealPath(storePath));
if (hash != *expectedHash)
state.debugThrowLastTrace(EvalError((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s",
*url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true)));
*url, expectedHash->to_string(HashFormat::Base32, true), hash.to_string(HashFormat::Base32, true)));
}
state.allowAndSetStorePathString(storePath, v);
@ -383,7 +392,7 @@ static RegisterPrimOp primop_fetchGit({
The URL of the repo.
- `name` (default: *basename of the URL*)
- `name` (default: `source`)
The name of the directory the repo should be exported to in the store.

8
src/libexpr/tests/error_traces.cc
View file

@ -310,7 +310,7 @@ namespace nix {
ASSERT_TRACE2("storePath true",
TypeError,
hintfmt("cannot coerce %s to a string", "a Boolean"),
hintfmt("while evaluating the first argument passed to builtins.storePath"));
hintfmt("while evaluating the first argument passed to 'builtins.storePath'"));
}
@ -378,12 +378,12 @@ namespace nix {
ASSERT_TRACE2("filterSource [] []",
TypeError,
hintfmt("cannot coerce %s to a string", "a list"),
hintfmt("while evaluating the second argument (the path to filter) passed to builtins.filterSource"));
hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'"));
ASSERT_TRACE2("filterSource [] \"foo\"",
EvalError,
hintfmt("string '%s' doesn't represent an absolute path", "foo"),
hintfmt("while evaluating the second argument (the path to filter) passed to builtins.filterSource"));
hintfmt("while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'"));
ASSERT_TRACE2("filterSource [] ./.",
TypeError,
@ -1084,7 +1084,7 @@ namespace nix {
ASSERT_TRACE1("hashString \"foo\" \"content\"",
UsageError,
hintfmt("unknown hash algorithm '%s'", "foo"));
hintfmt("unknown hash algorithm '%s', expect 'md5', 'sha1', 'sha256', or 'sha512'", "foo"));
ASSERT_TRACE2("hashString \"sha256\" {}",
TypeError,

2
src/libexpr/tests/json.cc
View file

@ -62,7 +62,7 @@ namespace nix {
// not supported by store 'dummy'" thrown in the test body.
TEST_F(JSONValueTest, DISABLED_Path) {
Value v;
v.mkPath("test");
v.mkPath(state.rootPath(CanonPath("/test")));
ASSERT_EQ(getJSONValue(v), "\"/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x\"");
}
} /* namespace nix */

15
src/libexpr/tests/libexpr.hh
View file

@ -103,14 +103,17 @@ namespace nix {
}
MATCHER_P(IsPathEq, p, fmt("Is a path equal to \"%1%\"", p)) {
if (arg.type() != nPath) {
*result_listener << "Expected a path got " << arg.type();
return false;
} else if (std::string_view(arg._path) != p) {
*result_listener << "Expected a path that equals \"" << p << "\" but got: " << arg.c_str();
if (arg.type() != nPath) {
*result_listener << "Expected a path got " << arg.type();
return false;
} else {
auto path = arg.path();
if (path.path != CanonPath(p)) {
*result_listener << "Expected a path that equals \"" << p << "\" but got: " << path.path;
return false;
}
return true;
}
return true;
}

17
src/libexpr/value.hh
View file

@ -189,7 +189,12 @@ public:
const char * c_str;
const char * * context; // must be in sorted order
} string;
const char * _path;
struct {
InputAccessor * accessor;
const char * path;
} _path;
Bindings * attrs;
struct {
size_t size;
@ -286,11 +291,12 @@ public:
void mkPath(const SourcePath & path);
inline void mkPath(const char * path)
inline void mkPath(InputAccessor * accessor, const char * path)
{
clearValue();
internalType = tPath;
_path = path;
_path.accessor = accessor;
_path.path = path;
}
inline void mkNull()
@ -437,7 +443,10 @@ public:
SourcePath path() const
{
assert(internalType == tPath);
return SourcePath{CanonPath(_path)};
return SourcePath {
.accessor = ref(_path.accessor->shared_from_this()),
.path = CanonPath(CanonPath::unchecked_t(), _path.path)
};
}
std::string_view string_view() const

34
src/libfetchers/fetchers.cc
View file

@ -84,16 +84,16 @@ std::string Input::to_string() const
return toURL().to_string();
}
bool Input::isDirect() const
{
return !scheme || scheme->isDirect(*this);
}
Attrs Input::toAttrs() const
{
return attrs;
}
bool Input::hasAllInfo() const
{
return getNarHash() && scheme && scheme->hasAllInfo(*this);
}
bool Input::operator ==(const Input & other) const
{
return attrs == other.attrs;
@ -109,7 +109,7 @@ bool Input::contains(const Input & other) const
return false;
}
std::pair<Tree, Input> Input::fetch(ref<Store> store) const
std::pair<StorePath, Input> Input::fetch(ref<Store> store) const
{
if (!scheme)
throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs()));
@ -117,7 +117,7 @@ std::pair<Tree, Input> Input::fetch(ref<Store> store) const
/* The tree may already be in the Nix store, or it could be
substituted (which is often faster than fetching from the
original source). So check that. */
if (hasAllInfo()) {
if (getNarHash()) {
try {
auto storePath = computeStorePath(*store);
@ -126,7 +126,7 @@ std::pair<Tree, Input> Input::fetch(ref<Store> store) const
debug("using substituted/cached input '%s' in '%s'",
to_string(), store->printStorePath(storePath));
return {Tree { .actualPath = store->toRealPath(storePath), .storePath = std::move(storePath) }, *this};
return {std::move(storePath), *this};
} catch (Error & e) {
debug("substitution of input '%s' failed: %s", to_string(), e.what());
}
@ -141,18 +141,16 @@ std::pair<Tree, Input> Input::fetch(ref<Store> store) const
}
}();
Tree tree {
.actualPath = store->toRealPath(storePath),
.storePath = storePath,
};
auto narHash = store->queryPathInfo(tree.storePath)->narHash;
input.attrs.insert_or_assign("narHash", narHash.to_string(SRI, true));
auto narHash = store->queryPathInfo(storePath)->narHash;
input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));
if (auto prevNarHash = getNarHash()) {
if (narHash != *prevNarHash)
throw Error((unsigned int) 102, "NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash.to_string(SRI, true));
to_string(),
store->printStorePath(storePath),
prevNarHash->to_string(HashFormat::SRI, true),
narHash.to_string(HashFormat::SRI, true));
}
if (auto prevLastModified = getLastModified()) {
@ -175,9 +173,7 @@ std::pair<Tree, Input> Input::fetch(ref<Store> store) const
input.locked = true;
assert(input.hasAllInfo());
return {std::move(tree), input};
return {std::move(storePath), input};
}
Input Input::applyOverrides(

58
src/libfetchers/fetchers.hh
View file

@ -13,12 +13,6 @@ namespace nix { class Store; }
namespace nix::fetchers {
struct Tree
{
Path actualPath;
StorePath storePath;
};
struct InputScheme;
/**
@ -35,7 +29,6 @@ struct Input
std::shared_ptr<InputScheme> scheme; // note: can be null
Attrs attrs;
bool locked = false;
bool direct = true;
/**
* path of the parent of this input, used for relative path resolution
@ -71,7 +64,7 @@ public:
* Check whether this is a "direct" input, that is, not
* one that goes through a registry.
*/
bool isDirect() const { return direct; }
bool isDirect() const;
/**
* Check whether this is a "locked" input, that is,
@ -79,24 +72,15 @@ public:
*/
bool isLocked() const { return locked; }
/**
* Check whether the input carries all necessary info required
* for cache insertion and substitution.
* These fields are used to uniquely identify cached trees
* within the "tarball TTL" window without necessarily
* indicating that the input's origin is unchanged.
*/
bool hasAllInfo() const;
bool operator ==(const Input & other) const;
bool contains(const Input & other) const;
/**
* Fetch the input into the Nix store, returning the location in
* the Nix store and the locked input.
* Fetch the entire input into the Nix store, returning the
* location in the Nix store and the locked input.
*/
std::pair<Tree, Input> fetch(ref<Store> store) const;
std::pair<StorePath, Input> fetch(ref<Store> store) const;
Input applyOverrides(
std::optional<std::string> ref,
@ -144,8 +128,6 @@ struct InputScheme
virtual ParsedURL toURL(const Input & input) const;
virtual bool hasAllInfo(const Input & input) const = 0;
virtual Input applyOverrides(
const Input & input,
std::optional<std::string> ref,
@ -163,37 +145,11 @@ struct InputScheme
* Is this `InputScheme` part of an experimental feature?
*/
virtual std::optional<ExperimentalFeature> experimentalFeature();
virtual bool isDirect(const Input & input) const
{ return true; }
};
void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);
struct DownloadFileResult
{
StorePath storePath;
std::string etag;
std::string effectiveUrl;
std::optional<std::string> immutableUrl;
};
DownloadFileResult downloadFile(
ref<Store> store,
const std::string & url,
const std::string & name,
bool locked,
const Headers & headers = {});
struct DownloadTarballResult
{
Tree tree;
time_t lastModified;
std::optional<std::string> immutableUrl;
};
DownloadTarballResult downloadTarball(
ref<Store> store,
const std::string & url,
const std::string & name,
bool locked,
const Headers & headers = {});
}

130
src/libfetchers/fs-input-accessor.cc Normal file
View file

@ -0,0 +1,130 @@
#include "fs-input-accessor.hh"
#include "posix-source-accessor.hh"
#include "store-api.hh"
namespace nix {
struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor
{
CanonPath root;
std::optional<std::set<CanonPath>> allowedPaths;
MakeNotAllowedError makeNotAllowedError;
FSInputAccessorImpl(
const CanonPath & root,
std::optional<std::set<CanonPath>> && allowedPaths,
MakeNotAllowedError && makeNotAllowedError)
: root(root)
, allowedPaths(std::move(allowedPaths))
, makeNotAllowedError(std::move(makeNotAllowedError))
{
}
void readFile(
const CanonPath & path,
Sink & sink,
std::function<void(uint64_t)> sizeCallback) override
{
auto absPath = makeAbsPath(path);
checkAllowed(absPath);
PosixSourceAccessor::readFile(absPath, sink, sizeCallback);
}
bool pathExists(const CanonPath & path) override
{
auto absPath = makeAbsPath(path);
return isAllowed(absPath) && PosixSourceAccessor::pathExists(absPath);
}
Stat lstat(const CanonPath & path) override
{
auto absPath = makeAbsPath(path);
checkAllowed(absPath);
return PosixSourceAccessor::lstat(absPath);
}
DirEntries readDirectory(const CanonPath & path) override
{
auto absPath = makeAbsPath(path);
checkAllowed(absPath);
DirEntries res;
for (auto & entry : PosixSourceAccessor::readDirectory(absPath))
if (isAllowed(absPath + entry.first))
res.emplace(entry);
return res;
}
std::string readLink(const CanonPath & path) override
{
auto absPath = makeAbsPath(path);
checkAllowed(absPath);
return PosixSourceAccessor::readLink(absPath);
}
CanonPath makeAbsPath(const CanonPath & path)
{
return root + path;
}
void checkAllowed(const CanonPath & absPath) override
{
if (!isAllowed(absPath))
throw makeNotAllowedError
? makeNotAllowedError(absPath)
: RestrictedPathError("access to path '%s' is forbidden", absPath);
}
bool isAllowed(const CanonPath & absPath)
{
if (!absPath.isWithin(root))
return false;
if (allowedPaths) {
auto p = absPath.removePrefix(root);
if (!p.isAllowed(*allowedPaths))
return false;
}
return true;
}
void allowPath(CanonPath path) override
{
if (allowedPaths)
allowedPaths->insert(std::move(path));
}
bool hasAccessControl() override
{
return (bool) allowedPaths;
}
std::optional<CanonPath> getPhysicalPath(const CanonPath & path) override
{
return makeAbsPath(path);
}
};
ref<FSInputAccessor> makeFSInputAccessor(
const CanonPath & root,
std::optional<std::set<CanonPath>> && allowedPaths,
MakeNotAllowedError && makeNotAllowedError)
{
return make_ref<FSInputAccessorImpl>(root, std::move(allowedPaths), std::move(makeNotAllowedError));
}
ref<FSInputAccessor> makeStorePathAccessor(
ref<Store> store,
const StorePath & storePath,
MakeNotAllowedError && makeNotAllowedError)
{
return makeFSInputAccessor(CanonPath(store->toRealPath(storePath)), {}, std::move(makeNotAllowedError));
}
SourcePath getUnfilteredRootPath(CanonPath path)
{
static auto rootFS = makeFSInputAccessor(CanonPath::root);
return {rootFS, path};
}
}

33
src/libfetchers/fs-input-accessor.hh Normal file
View file

@ -0,0 +1,33 @@
#pragma once
#include "input-accessor.hh"
namespace nix {
class StorePath;
class Store;
struct FSInputAccessor : InputAccessor
{
virtual void checkAllowed(const CanonPath & absPath) = 0;
virtual void allowPath(CanonPath path) = 0;
virtual bool hasAccessControl() = 0;
};
typedef std::function<RestrictedPathError(const CanonPath & path)> MakeNotAllowedError;
ref<FSInputAccessor> makeFSInputAccessor(
const CanonPath & root,
std::optional<std::set<CanonPath>> && allowedPaths = {},
MakeNotAllowedError && makeNotAllowedError = {});
ref<FSInputAccessor> makeStorePathAccessor(
ref<Store> store,
const StorePath & storePath,
MakeNotAllowedError && makeNotAllowedError = {});
SourcePath getUnfilteredRootPath(CanonPath path);
}

13
src/libfetchers/git.cc
View file

@ -46,7 +46,7 @@ bool touchCacheFile(const Path & path, time_t touch_time)
Path getCachePath(std::string_view key)
{
return getCacheDir() + "/nix/gitv3/" +
hashString(htSHA256, key).to_string(Base32, false);
hashString(htSHA256, key).to_string(HashFormat::Base32, false);
}
// Returns the name of the HEAD branch.
@ -322,15 +322,6 @@ struct GitInputScheme : InputScheme
return url;
}
bool hasAllInfo(const Input & input) const override
{
bool maybeDirty = !input.getRef();
bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
return
maybeGetIntAttr(input.attrs, "lastModified")
&& (shallow || maybeDirty || maybeGetIntAttr(input.attrs, "revCount"));
}
Input applyOverrides(
const Input & input,
std::optional<std::string> ref,
@ -418,7 +409,7 @@ struct GitInputScheme : InputScheme
auto checkHashType = [&](const std::optional<Hash> & hash)
{
if (hash.has_value() && !(hash->type == htSHA1 || hash->type == htSHA256))
throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(Base16, true));
throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(HashFormat::Base16, true));
};
auto getLockedAttrs = [&]()

23
src/libfetchers/github.cc
View file

@ -7,6 +7,7 @@
#include "git.hh"
#include "fetchers.hh"
#include "fetch-settings.hh"
#include "tarball.hh"
#include <optional>
#include <nlohmann/json.hpp>
@ -125,18 +126,13 @@ struct GitArchiveInputScheme : InputScheme
auto path = owner + "/" + repo;
assert(!(ref && rev));
if (ref) path += "/" + *ref;
if (rev) path += "/" + rev->to_string(Base16, false);
if (rev) path += "/" + rev->to_string(HashFormat::Base16, false);
return ParsedURL {
.scheme = type(),
.path = path,
};
}
bool hasAllInfo(const Input & input) const override
{
return input.getRev() && maybeGetIntAttr(input.attrs, "lastModified");
}
Input applyOverrides(
const Input & _input,
std::optional<std::string> ref,
@ -218,10 +214,15 @@ struct GitArchiveInputScheme : InputScheme
{"rev", rev->gitRev()},
{"lastModified", uint64_t(result.lastModified)}
},
result.tree.storePath,
result.storePath,
true);
return {result.tree.storePath, input};
return {result.storePath, input};
}
std::optional<ExperimentalFeature> experimentalFeature() override
{
return Xp::Flakes;
}
};
@ -291,7 +292,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
: "https://api.%s/repos/%s/%s/tarball/%s";
const auto url = fmt(urlFmt, host, getOwner(input), getRepo(input),
input.getRev()->to_string(Base16, false));
input.getRev()->to_string(HashFormat::Base16, false));
return DownloadUrl { url, headers };
}
@ -357,7 +358,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s",
host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
input.getRev()->to_string(Base16, false));
input.getRev()->to_string(HashFormat::Base16, false));
Headers headers = makeHeadersWithAuthTokens(host);
return DownloadUrl { url, headers };
@ -444,7 +445,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht");
auto url = fmt("https://%s/%s/%s/archive/%s.tar.gz",
host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
input.getRev()->to_string(Base16, false));
input.getRev()->to_string(HashFormat::Base16, false));
Headers headers = makeHeadersWithAuthTokens(host);
return DownloadUrl { url, headers };

10
src/libfetchers/indirect.cc
View file

@ -41,7 +41,6 @@ struct IndirectInputScheme : InputScheme
// FIXME: forbid query params?
Input input;
input.direct = false;
input.attrs.insert_or_assign("type", "indirect");
input.attrs.insert_or_assign("id", id);
if (rev) input.attrs.insert_or_assign("rev", rev->gitRev());
@ -63,7 +62,6 @@ struct IndirectInputScheme : InputScheme
throw BadURL("'%s' is not a valid flake ID", id);
Input input;
input.direct = false;
input.attrs = attrs;
return input;
}
@ -78,11 +76,6 @@ struct IndirectInputScheme : InputScheme
return url;
}
bool hasAllInfo(const Input & input) const override
{
return false;
}
Input applyOverrides(
const Input & _input,
std::optional<std::string> ref,
@ -103,6 +96,9 @@ struct IndirectInputScheme : InputScheme
{
return Xp::Flakes;
}
bool isDirect(const Input & input) const override
{ return false; }
};
static auto rIndirectInputScheme = OnStartup([] { registerInputScheme(std::make_unique<IndirectInputScheme>()); });

92
src/libfetchers/input-accessor.cc
View file

@ -3,12 +3,52 @@
namespace nix {
StorePath InputAccessor::fetchToStore(
ref<Store> store,
const CanonPath & path,
std::string_view name,
FileIngestionMethod method,
PathFilter * filter,
RepairFlag repair)
{
Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", showPath(path)));
auto source = sinkToSource([&](Sink & sink) {
if (method == FileIngestionMethod::Recursive)
dumpPath(path, sink, filter ? *filter : defaultPathFilter);
else
readFile(path, sink);
});
auto storePath =
settings.readOnlyMode
? store->computeStorePathFromDump(*source, name, method, htSHA256).first
: store->addToStoreFromDump(*source, name, method, htSHA256, repair);
return storePath;
}
SourcePath InputAccessor::root()
{
return {ref(shared_from_this()), CanonPath::root};
}
std::ostream & operator << (std::ostream & str, const SourcePath & path)
{
str << path.to_string();
return str;
}
StorePath SourcePath::fetchToStore(
ref<Store> store,
std::string_view name,
FileIngestionMethod method,
PathFilter * filter,
RepairFlag repair) const
{
return accessor->fetchToStore(store, path, name, method, filter, repair);
}
std::string_view SourcePath::baseName() const
{
return path.baseName().value_or("source");
@ -18,60 +58,12 @@ SourcePath SourcePath::parent() const
{
auto p = path.parent();
assert(p);
return std::move(*p);
}
InputAccessor::Stat SourcePath::lstat() const
{
auto st = nix::lstat(path.abs());
return InputAccessor::Stat {
.type =
S_ISREG(st.st_mode) ? InputAccessor::tRegular :
S_ISDIR(st.st_mode) ? InputAccessor::tDirectory :
S_ISLNK(st.st_mode) ? InputAccessor::tSymlink :
InputAccessor::tMisc,
.isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR
};
}
std::optional<InputAccessor::Stat> SourcePath::maybeLstat() const
{
// FIXME: merge these into one operation.
if (!pathExists())
return {};
return lstat();
}
InputAccessor::DirEntries SourcePath::readDirectory() const
{
InputAccessor::DirEntries res;
for (auto & entry : nix::readDirectory(path.abs())) {
std::optional<InputAccessor::Type> type;
switch (entry.type) {
case DT_REG: type = InputAccessor::Type::tRegular; break;
case DT_LNK: type = InputAccessor::Type::tSymlink; break;
case DT_DIR: type = InputAccessor::Type::tDirectory; break;
}
res.emplace(entry.name, type);
}
return res;
}
StorePath SourcePath::fetchToStore(
ref<Store> store,
std::string_view name,
PathFilter * filter,
RepairFlag repair) const
{
return
settings.readOnlyMode
? store->computeStorePathForPath(name, path.abs(), FileIngestionMethod::Recursive, htSHA256, filter ? *filter : defaultPathFilter).first
: store->addToStore(name, path.abs(), FileIngestionMethod::Recursive, htSHA256, filter ? *filter : defaultPathFilter, repair);
return {accessor, std::move(*p)};
}
SourcePath SourcePath::resolveSymlinks() const
{
SourcePath res(CanonPath::root);
auto res = accessor->root();
int linksAllowed = 1024;

78
src/libfetchers/input-accessor.hh
View file

@ -1,40 +1,39 @@
#pragma once
#include "source-accessor.hh"
#include "ref.hh"
#include "types.hh"
#include "archive.hh"
#include "canon-path.hh"
#include "repair-flag.hh"
#include "content-address.hh"
namespace nix {
MakeError(RestrictedPathError, Error);
struct SourcePath;
class StorePath;
class Store;
struct InputAccessor
struct InputAccessor : SourceAccessor, std::enable_shared_from_this<InputAccessor>
{
enum Type {
tRegular, tSymlink, tDirectory,
/**
Any other node types that may be encountered on the file system, such as device nodes, sockets, named pipe, and possibly even more exotic things.
Responsible for `"unknown"` from `builtins.readFileType "/dev/null"`.
Unlike `DT_UNKNOWN`, this must not be used for deferring the lookup of types.
*/
tMisc
};
struct Stat
/**
* Return the maximum last-modified time of the files in this
* tree, if available.
*/
virtual std::optional<time_t> getLastModified()
{
Type type = tMisc;
//uint64_t fileSize = 0; // regular files only
bool isExecutable = false; // regular files only
};
return std::nullopt;
}
typedef std::optional<Type> DirEntry;
StorePath fetchToStore(
ref<Store> store,
const CanonPath & path,
std::string_view name = "source",
FileIngestionMethod method = FileIngestionMethod::Recursive,
PathFilter * filter = nullptr,
RepairFlag repair = NoRepair);
typedef std::map<std::string, DirEntry> DirEntries;
SourcePath root();
};
/**
@ -45,12 +44,9 @@ struct InputAccessor
*/
struct SourcePath
{
ref<InputAccessor> accessor;
CanonPath path;
SourcePath(CanonPath path)
: path(std::move(path))
{ }
std::string_view baseName() const;
/**
@ -64,39 +60,42 @@ struct SourcePath
* return its contents; otherwise throw an error.
*/
std::string readFile() const
{ return nix::readFile(path.abs()); }
{ return accessor->readFile(path); }
/**
* Return whether this `SourcePath` denotes a file (of any type)
* that exists
*/
bool pathExists() const
{ return nix::pathExists(path.abs()); }
{ return accessor->pathExists(path); }
/**
* Return stats about this `SourcePath`, or throw an exception if
* it doesn't exist.
*/
InputAccessor::Stat lstat() const;
InputAccessor::Stat lstat() const
{ return accessor->lstat(path); }
/**
* Return stats about this `SourcePath`, or std::nullopt if it
* doesn't exist.
*/
std::optional<InputAccessor::Stat> maybeLstat() const;
std::optional<InputAccessor::Stat> maybeLstat() const
{ return accessor->maybeLstat(path); }
/**
* If this `SourcePath` denotes a directory (not a symlink),
* return its directory entries; otherwise throw an error.
*/
InputAccessor::DirEntries readDirectory() const;
InputAccessor::DirEntries readDirectory() const
{ return accessor->readDirectory(path); }
/**
* If this `SourcePath` denotes a symlink, return its target;
* otherwise throw an error.
*/
std::string readLink() const
{ return nix::readLink(path.abs()); }
{ return accessor->readLink(path); }
/**
* Dump this `SourcePath` to `sink` as a NAR archive.
@ -104,7 +103,7 @@ struct SourcePath
void dumpPath(
Sink & sink,
PathFilter & filter = defaultPathFilter) const
{ return nix::dumpPath(path.abs(), sink, filter); }
{ return accessor->dumpPath(path, sink, filter); }
/**
* Copy this `SourcePath` to the Nix store.
@ -112,6 +111,7 @@ struct SourcePath
StorePath fetchToStore(
ref<Store> store,
std::string_view name = "source",
FileIngestionMethod method = FileIngestionMethod::Recursive,
PathFilter * filter = nullptr,
RepairFlag repair = NoRepair) const;
@ -120,7 +120,7 @@ struct SourcePath
* it has a physical location.
*/
std::optional<CanonPath> getPhysicalPath() const
{ return path; }
{ return accessor->getPhysicalPath(path); }
std::string to_string() const
{ return path.abs(); }
@ -129,7 +129,7 @@ struct SourcePath
* Append a `CanonPath` to this path.
*/
SourcePath operator + (const CanonPath & x) const
{ return {path + x}; }
{ return {accessor, path + x}; }
/**
* Append a single component `c` to this path. `c` must not
@ -137,21 +137,21 @@ struct SourcePath
* and `c`.
*/
SourcePath operator + (std::string_view c) const
{ return {path + c}; }
{ return {accessor, path + c}; }
bool operator == (const SourcePath & x) const
{
return path == x.path;
return std::tie(accessor, path) == std::tie(x.accessor, x.path);
}
bool operator != (const SourcePath & x) const
{
return path != x.path;
return std::tie(accessor, path) != std::tie(x.accessor, x.path);
}
bool operator < (const SourcePath & x) const
{
return path < x.path;
return std::tie(accessor, path) < std::tie(x.accessor, x.path);
}
/**

54
src/libfetchers/memory-input-accessor.cc Normal file
View file

@ -0,0 +1,54 @@
#include "memory-input-accessor.hh"
namespace nix {
struct MemoryInputAccessorImpl : MemoryInputAccessor
{
std::map<CanonPath, std::string> files;
std::string readFile(const CanonPath & path) override
{
auto i = files.find(path);
if (i == files.end())
throw Error("file '%s' does not exist", path);
return i->second;
}
bool pathExists(const CanonPath & path) override
{
auto i = files.find(path);
return i != files.end();
}
Stat lstat(const CanonPath & path) override
{
auto i = files.find(path);
if (i != files.end())
return Stat { .type = tRegular, .isExecutable = false };
throw Error("file '%s' does not exist", path);
}
DirEntries readDirectory(const CanonPath & path) override
{
return {};
}
std::string readLink(const CanonPath & path) override
{
throw UnimplementedError("MemoryInputAccessor::readLink");
}
SourcePath addFile(CanonPath path, std::string && contents) override
{
files.emplace(path, std::move(contents));
return {ref(shared_from_this()), std::move(path)};
}
};
ref<MemoryInputAccessor> makeMemoryInputAccessor()
{
return make_ref<MemoryInputAccessorImpl>();
}
}

15
src/libfetchers/memory-input-accessor.hh Normal file
View file

@ -0,0 +1,15 @@
#include "input-accessor.hh"
namespace nix {
/**
* An input accessor for an in-memory file system.
*/
struct MemoryInputAccessor : InputAccessor
{
virtual SourcePath addFile(CanonPath path, std::string && contents) = 0;
};
ref<MemoryInputAccessor> makeMemoryInputAccessor();
}

11
src/libfetchers/mercurial.cc
View file

@ -98,13 +98,6 @@ struct MercurialInputScheme : InputScheme
return url;
}
bool hasAllInfo(const Input & input) const override
{
// FIXME: ugly, need to distinguish between dirty and clean
// default trees.
return input.getRef() == "default" || maybeGetIntAttr(input.attrs, "revCount");
}
Input applyOverrides(
const Input & input,
std::optional<std::string> ref,
@ -206,7 +199,7 @@ struct MercurialInputScheme : InputScheme
auto checkHashType = [&](const std::optional<Hash> & hash)
{
if (hash.has_value() && hash->type != htSHA1)
throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", hash->to_string(Base16, true));
throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", hash->to_string(HashFormat::Base16, true));
};
@ -252,7 +245,7 @@ struct MercurialInputScheme : InputScheme
}
}
Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(Base32, false));
Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(HashFormat::Base32, false));
/* If this is a commit hash that we already have, we don't
have to pull again. */

10
src/libfetchers/path.cc
View file

@ -66,11 +66,6 @@ struct PathInputScheme : InputScheme
};
}
bool hasAllInfo(const Input & input) const override
{
return true;
}
std::optional<Path> getSourcePath(const Input & input) override
{
return getStrAttr(input.attrs, "path");
@ -125,6 +120,11 @@ struct PathInputScheme : InputScheme
return {std::move(*storePath), input};
}
std::optional<ExperimentalFeature> experimentalFeature() override
{
return Xp::Flakes;
}
};
static auto rPathInputScheme = OnStartup([] { registerInputScheme(std::make_unique<PathInputScheme>()); });

2
src/libfetchers/registry.cc
View file

@ -1,5 +1,5 @@
#include "registry.hh"
#include "fetchers.hh"
#include "tarball.hh"
#include "util.hh"
#include "globals.hh"
#include "store-api.hh"

15
src/libfetchers/tarball.cc
View file

@ -1,3 +1,4 @@
#include "tarball.hh"
#include "fetchers.hh"
#include "cache.hh"
#include "filetransfer.hh"
@ -133,7 +134,7 @@ DownloadTarballResult downloadTarball(
if (cached && !cached->expired)
return {
.tree = Tree { .actualPath = store->toRealPath(cached->storePath), .storePath = std::move(cached->storePath) },
.storePath = std::move(cached->storePath),
.lastModified = (time_t) getIntAttr(cached->infoAttrs, "lastModified"),
.immutableUrl = maybeGetStrAttr(cached->infoAttrs, "immutableUrl"),
};
@ -174,7 +175,7 @@ DownloadTarballResult downloadTarball(
locked);
return {
.tree = Tree { .actualPath = store->toRealPath(*unpackedStorePath), .storePath = std::move(*unpackedStorePath) },
.storePath = std::move(*unpackedStorePath),
.lastModified = lastModified,
.immutableUrl = res.immutableUrl,
};
@ -250,15 +251,9 @@ struct CurlInputScheme : InputScheme
// NAR hashes are preferred over file hashes since tar/zip
// files don't have a canonical representation.
if (auto narHash = input.getNarHash())
url.query.insert_or_assign("narHash", narHash->to_string(SRI, true));
url.query.insert_or_assign("narHash", narHash->to_string(HashFormat::SRI, true));
return url;
}
bool hasAllInfo(const Input & input) const override
{
return true;
}
};
struct FileInputScheme : CurlInputScheme
@ -313,7 +308,7 @@ struct TarballInputScheme : CurlInputScheme
if (result.lastModified && !input.attrs.contains("lastModified"))
input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified));
return {result.tree.storePath, std::move(input)};
return {result.storePath, std::move(input)};
}
};

43
src/libfetchers/tarball.hh Normal file
View file

@ -0,0 +1,43 @@
#pragma once
#include "types.hh"
#include "path.hh"
#include <optional>
namespace nix {
class Store;
}
namespace nix::fetchers {
struct DownloadFileResult
{
StorePath storePath;
std::string etag;
std::string effectiveUrl;
std::optional<std::string> immutableUrl;
};
DownloadFileResult downloadFile(
ref<Store> store,
const std::string & url,
const std::string & name,
bool locked,
const Headers & headers = {});
struct DownloadTarballResult
{
StorePath storePath;
time_t lastModified;
std::optional<std::string> immutableUrl;
};
DownloadTarballResult downloadTarball(
ref<Store> store,
const std::string & url,
const std::string & name,
bool locked,
const Headers & headers = {});
}

9
src/libmain/common-args.cc
View file

@ -1,4 +1,5 @@
#include "common-args.hh"
#include "args/root.hh"
#include "globals.hh"
#include "loggers.hh"
@ -34,21 +35,21 @@ MixCommonArgs::MixCommonArgs(const std::string & programName)
.description = "Set the Nix configuration setting *name* to *value* (overriding `nix.conf`).",
.category = miscCategory,
.labels = {"name", "value"},
.handler = {[](std::string name, std::string value) {
.handler = {[this](std::string name, std::string value) {
try {
globalConfig.set(name, value);
} catch (UsageError & e) {
if (!completions)
if (!getRoot().completions)
warn(e.what());
}
}},
.completer = [](size_t index, std::string_view prefix) {
.completer = [](AddCompletions & completions, size_t index, std::string_view prefix) {
if (index == 0) {
std::map<std::string, Config::SettingInfo> settings;
globalConfig.getSettings(settings);
for (auto & s : settings)
if (hasPrefix(s.first, prefix))
completions->add(s.first, fmt("Set the `%s` setting.", s.first));
completions.add(s.first, fmt("Set the `%s` setting.", s.first));
}
}
});

3
src/libmain/shared.hh
View file

@ -3,6 +3,7 @@
#include "util.hh"
#include "args.hh"
#include "args/root.hh"
#include "common-args.hh"
#include "path.hh"
#include "derived-path.hh"
@ -66,7 +67,7 @@ template<class N> N getIntArg(const std::string & opt,
}
struct LegacyArgs : public MixCommonArgs
struct LegacyArgs : public MixCommonArgs, public RootArgs
{
std::function<bool(Strings::iterator & arg, const Strings::iterator & end)> parseArg;

2
src/libstore/binary-cache-store.cc
View file

@ -164,7 +164,7 @@ ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
auto [fileHash, fileSize] = fileHashSink.finish();
narInfo->fileHash = fileHash;
narInfo->fileSize = fileSize;
narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar"
narInfo->url = "nar/" + narInfo->fileHash->to_string(HashFormat::Base32, false) + ".nar"
+ (compression == "xz" ? ".xz" :
compression == "bzip2" ? ".bz2" :
compression == "zstd" ? ".zst" :

2
src/libstore/build/derivation-goal.cc
View file

@ -561,7 +561,7 @@ void DerivationGoal::inputsRealised()
attempt = fullDrv.tryResolve(worker.store);
}
assert(attempt);
Derivation drvResolved { *std::move(attempt) };
Derivation drvResolved { std::move(*attempt) };
auto pathResolved = writeDerivation(worker.store, drvResolved);

151
src/libstore/build/local-derivation-goal.cc
View file

@ -227,7 +227,7 @@ void LocalDerivationGoal::tryLocalBuild()
if (!buildUser) {
if (!actLock)
actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath))));
fmt("waiting for a free build user ID for '%s'", yellowtxt(worker.store.printStorePath(drvPath))));
worker.waitForAWhile(shared_from_this());
return;
}
@ -386,27 +386,27 @@ void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck()
cleanupPostOutputsRegisteredModeCheck();
}
#if __linux__
static void linkOrCopy(const Path & from, const Path & to)
{
if (link(from.c_str(), to.c_str()) == -1) {
/* Hard-linking fails if we exceed the maximum link count on a
file (e.g. 32000 of ext3), which is quite possible after a
'nix-store --optimise'. FIXME: actually, why don't we just
bind-mount in this case?
It can also fail with EPERM in BeegFS v7 and earlier versions
or fail with EXDEV in OpenAFS
which don't allow hard-links to other directories */
if (errno != EMLINK && errno != EPERM && errno != EXDEV)
throw SysError("linking '%s' to '%s'", to, from);
copyPath(from, to);
static void doBind(const Path & source, const Path & target, bool optional = false) {
debug("bind mounting '%1%' to '%2%'", source, target);
struct stat st;
if (stat(source.c_str(), &st) == -1) {
if (optional && errno == ENOENT)
return;
else
throw SysError("getting attributes of path '%1%'", source);
}
}
if (S_ISDIR(st.st_mode))
createDirs(target);
else {
createDirs(dirOf(target));
writeFile(target, "");
}
if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1)
throw SysError("bind mount from '%1%' to '%2%' failed", source, target);
};
#endif
void LocalDerivationGoal::startBuilder()
{
if ((buildUser && buildUser->getUIDCount() != 1)
@ -581,7 +581,7 @@ void LocalDerivationGoal::startBuilder()
/* Allow a user-configurable set of directories from the
host file system. */
dirsInChroot.clear();
pathsInChroot.clear();
for (auto i : settings.sandboxPaths.get()) {
if (i.empty()) continue;
@ -592,19 +592,19 @@ void LocalDerivationGoal::startBuilder()
}
size_t p = i.find('=');
if (p == std::string::npos)
dirsInChroot[i] = {i, optional};
pathsInChroot[i] = {i, optional};
else
dirsInChroot[i.substr(0, p)] = {i.substr(p + 1), optional};
pathsInChroot[i.substr(0, p)] = {i.substr(p + 1), optional};
}
if (hasPrefix(worker.store.storeDir, tmpDirInSandbox))
{
throw Error("`sandbox-build-dir` must not contain the storeDir");
}
dirsInChroot[tmpDirInSandbox] = tmpDir;
pathsInChroot[tmpDirInSandbox] = tmpDir;
/* Add the closure of store paths to the chroot. */
StorePathSet closure;
for (auto & i : dirsInChroot)
for (auto & i : pathsInChroot)
try {
if (worker.store.isInStore(i.second.source))
worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure);
@ -615,7 +615,7 @@ void LocalDerivationGoal::startBuilder()
}
for (auto & i : closure) {
auto p = worker.store.printStorePath(i);
dirsInChroot.insert_or_assign(p, p);
pathsInChroot.insert_or_assign(p, p);
}
PathSet allowedPaths = settings.allowedImpureHostPrefixes;
@ -643,7 +643,7 @@ void LocalDerivationGoal::startBuilder()
/* Allow files in __impureHostDeps to be missing; e.g.
macOS 11+ has no /usr/lib/libSystem*.dylib */
dirsInChroot[i] = {i, true};
pathsInChroot[i] = {i, true};
}
#if __linux__
@ -711,15 +711,12 @@ void LocalDerivationGoal::startBuilder()
for (auto & i : inputPaths) {
auto p = worker.store.printStorePath(i);
Path r = worker.store.toRealPath(p);
if (S_ISDIR(lstat(r).st_mode))
dirsInChroot.insert_or_assign(p, r);
else
linkOrCopy(r, chrootRootDir + p);
pathsInChroot.insert_or_assign(p, r);
}
/* If we're repairing, checking or rebuilding part of a
multiple-outputs derivation, it's possible that we're
rebuilding a path that is in settings.dirsInChroot
rebuilding a path that is in settings.sandbox-paths
(typically the dependencies of /bin/sh). Throw them
out. */
for (auto & i : drv->outputsAndOptPaths(worker.store)) {
@ -729,7 +726,7 @@ void LocalDerivationGoal::startBuilder()
is already in the sandbox, so we don't need to worry about
removing it. */
if (i.second.second)
dirsInChroot.erase(worker.store.printStorePath(*i.second.second));
pathsInChroot.erase(worker.store.printStorePath(*i.second.second));
}
if (cgroup) {
@ -787,9 +784,9 @@ void LocalDerivationGoal::startBuilder()
} else {
auto p = line.find('=');
if (p == std::string::npos)
dirsInChroot[line] = line;
pathsInChroot[line] = line;
else
dirsInChroot[line.substr(0, p)] = line.substr(p + 1);
pathsInChroot[line.substr(0, p)] = line.substr(p + 1);
}
}
}
@ -1066,7 +1063,7 @@ void LocalDerivationGoal::initTmpDir() {
env[i.first] = i.second;
} else {
auto hash = hashString(htSHA256, i.first);
std::string fn = ".attr-" + hash.to_string(Base32, false);
std::string fn = ".attr-" + hash.to_string(HashFormat::Base32, false);
Path p = tmpDir + "/" + fn;
writeFile(p, rewriteStrings(i.second, inputRewrites));
chownToBuilder(p);
@ -1565,41 +1562,32 @@ void LocalDerivationGoal::addDependency(const StorePath & path)
Path source = worker.store.Store::toRealPath(path);
Path target = chrootRootDir + worker.store.printStorePath(path);
debug("bind-mounting %s -> %s", target, source);
if (pathExists(target))
// There is a similar debug message in doBind, so only run it in this block to not have double messages.
debug("bind-mounting %s -> %s", target, source);
throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path));
auto st = lstat(source);
/* Bind-mount the path into the sandbox. This requires
entering its mount namespace, which is not possible
in multithreaded programs. So we do this in a
child process.*/
Pid child(startProcess([&]() {
if (S_ISDIR(st.st_mode)) {
if (usingUserNamespace && (setns(sandboxUserNamespace.get(), 0) == -1))
throw SysError("entering sandbox user namespace");
/* Bind-mount the path into the sandbox. This requires
entering its mount namespace, which is not possible
in multithreaded programs. So we do this in a
child process.*/
Pid child(startProcess([&]() {
if (setns(sandboxMountNamespace.get(), 0) == -1)
throw SysError("entering sandbox mount namespace");
if (usingUserNamespace && (setns(sandboxUserNamespace.get(), 0) == -1))
throw SysError("entering sandbox user namespace");
doBind(source, target);
if (setns(sandboxMountNamespace.get(), 0) == -1)
throw SysError("entering sandbox mount namespace");
_exit(0);
}));
createDirs(target);
if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1)
throw SysError("bind mount from '%s' to '%s' failed", source, target);
_exit(0);
}));
int status = child.wait();
if (status != 0)
throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path));
} else
linkOrCopy(source, target);
int status = child.wait();
if (status != 0)
throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path));
#else
throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox",
@ -1789,7 +1777,7 @@ void LocalDerivationGoal::runChild()
/* Set up a nearly empty /dev, unless the user asked to
bind-mount the host /dev. */
Strings ss;
if (dirsInChroot.find("/dev") == dirsInChroot.end()) {
if (pathsInChroot.find("/dev") == pathsInChroot.end()) {
createDirs(chrootRootDir + "/dev/shm");
createDirs(chrootRootDir + "/dev/pts");
ss.push_back("/dev/full");
@ -1824,34 +1812,15 @@ void LocalDerivationGoal::runChild()
ss.push_back(path);
if (settings.caFile != "")
dirsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", settings.caFile, true);
pathsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", settings.caFile, true);
}
for (auto & i : ss) dirsInChroot.emplace(i, i);
for (auto & i : ss) pathsInChroot.emplace(i, i);
/* Bind-mount all the directories from the "host"
filesystem that we want in the chroot
environment. */
auto doBind = [&](const Path & source, const Path & target, bool optional = false) {
debug("bind mounting '%1%' to '%2%'", source, target);
struct stat st;
if (stat(source.c_str(), &st) == -1) {
if (optional && errno == ENOENT)
return;
else
throw SysError("getting attributes of path '%1%'", source);
}
if (S_ISDIR(st.st_mode))
createDirs(target);
else {
createDirs(dirOf(target));
writeFile(target, "");
}
if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1)
throw SysError("bind mount from '%1%' to '%2%' failed", source, target);
};
for (auto & i : dirsInChroot) {
for (auto & i : pathsInChroot) {
if (i.second.source == "/proc") continue; // backwards compatibility
#if HAVE_EMBEDDED_SANDBOX_SHELL
@ -1892,7 +1861,7 @@ void LocalDerivationGoal::runChild()
if /dev/ptx/ptmx exists). */
if (pathExists("/dev/pts/ptmx") &&
!pathExists(chrootRootDir + "/dev/ptmx")
&& !dirsInChroot.count("/dev/pts"))
&& !pathsInChroot.count("/dev/pts"))
{
if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0)
{
@ -2027,7 +1996,7 @@ void LocalDerivationGoal::runChild()
/* We build the ancestry before adding all inputPaths to the store because we know they'll
all have the same parents (the store), and there might be lots of inputs. This isn't
particularly efficient... I doubt it'll be a bottleneck in practice */
for (auto & i : dirsInChroot) {
for (auto & i : pathsInChroot) {
Path cur = i.first;
while (cur.compare("/") != 0) {
cur = dirOf(cur);
@ -2035,7 +2004,7 @@ void LocalDerivationGoal::runChild()
}
}
/* And we want the store in there regardless of how empty dirsInChroot. We include the innermost
/* And we want the store in there regardless of how empty pathsInChroot. We include the innermost
path component this time, since it's typically /nix/store and we care about that. */
Path cur = worker.store.storeDir;
while (cur.compare("/") != 0) {
@ -2046,7 +2015,7 @@ void LocalDerivationGoal::runChild()
/* Add all our input paths to the chroot */
for (auto & i : inputPaths) {
auto p = worker.store.printStorePath(i);
dirsInChroot[p] = p;
pathsInChroot[p] = p;
}
/* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */
@ -2077,7 +2046,7 @@ void LocalDerivationGoal::runChild()
without file-write* allowed, access() incorrectly returns EPERM
*/
sandboxProfile += "(allow file-read* file-write* process-exec\n";
for (auto & i : dirsInChroot) {
for (auto & i : pathsInChroot) {
if (i.first != i.second.source)
throw Error(
"can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin",
@ -2521,7 +2490,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
ValidPathInfo newInfo0 {
worker.store,
outputPathName(drv->name, outputName),
*std::move(optCA),
std::move(*optCA),
Hash::dummy,
};
if (*scratchPath != newInfo0.path) {
@ -2583,8 +2552,8 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
delayedException = std::make_exception_ptr(
BuildError("hash mismatch in fixed-output derivation '%s':\n specified: %s\n got: %s",
worker.store.printStorePath(drvPath),
wanted.to_string(SRI, true),
got.to_string(SRI, true)));
wanted.to_string(HashFormat::SRI, true),
got.to_string(HashFormat::SRI, true)));
}
if (!newInfo0.references.empty())
delayedException = std::make_exception_ptr(

12
src/libstore/build/local-derivation-goal.hh
View file

@ -86,8 +86,8 @@ struct LocalDerivationGoal : public DerivationGoal
: source(source), optional(optional)
{ }
};
typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
DirsInChroot dirsInChroot;
typedef map<Path, ChrootPath> PathsInChroot; // maps target path to source path
PathsInChroot pathsInChroot;
typedef map<std::string, std::string> Environment;
Environment env;
@ -120,14 +120,6 @@ struct LocalDerivationGoal : public DerivationGoal
*/
OutputPathMap scratchOutputs;
/**
* Path registration info from the previous round, if we're
* building multiple times. Since this contains the hash, it
* allows us to compare whether two rounds produced the same
* result.
*/
std::map<Path, ValidPathInfo> prevInfos;
uid_t sandboxUid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 1000 : 0) : buildUser->getUID(); }
gid_t sandboxGid() { return usingUserNamespace ? (!buildUser || buildUser->getUIDCount() == 1 ? 100 : 0) : buildUser->getGID(); }

22
src/libstore/builtins/buildenv.cc
View file

@ -174,15 +174,19 @@ void builtinBuildenv(const BasicDerivation & drv)
/* Convert the stuff we get from the environment back into a
* coherent data type. */
Packages pkgs;
auto derivations = tokenizeString<Strings>(getAttr("derivations"));
while (!derivations.empty()) {
/* !!! We're trusting the caller to structure derivations env var correctly */
auto active = derivations.front(); derivations.pop_front();
auto priority = stoi(derivations.front()); derivations.pop_front();
auto outputs = stoi(derivations.front()); derivations.pop_front();
for (auto n = 0; n < outputs; n++) {
auto path = derivations.front(); derivations.pop_front();
pkgs.emplace_back(path, active != "false", priority);
{
auto derivations = tokenizeString<Strings>(getAttr("derivations"));
auto itemIt = derivations.begin();
while (itemIt != derivations.end()) {
/* !!! We're trusting the caller to structure derivations env var correctly */
const bool active = "false" != *itemIt++;
const int priority = stoi(*itemIt++);
const size_t outputs = stoul(*itemIt++);
for (size_t n {0}; n < outputs; n++) {
pkgs.emplace_back(std::move(*itemIt++), active, priority);
}
}
}

2
src/libstore/builtins/fetchurl.cc
View file

@ -65,7 +65,7 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
std::optional<HashType> ht = parseHashTypeOpt(getAttr("outputHashAlgo"));
Hash h = newHashAllowEmpty(getAttr("outputHash"), ht);
fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false));
fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(HashFormat::Base16, false));
return;
} catch (Error & e) {
debug(e.what());

17
src/libstore/content-address.cc
View file

@ -29,12 +29,13 @@ std::string ContentAddressMethod::renderPrefix() const
ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m)
{
ContentAddressMethod method = FileIngestionMethod::Flat;
if (splitPrefix(m, "r:"))
method = FileIngestionMethod::Recursive;
else if (splitPrefix(m, "text:"))
method = TextIngestionMethod {};
return method;
if (splitPrefix(m, "r:")) {
return FileIngestionMethod::Recursive;
}
else if (splitPrefix(m, "text:")) {
return TextIngestionMethod {};
}
return FileIngestionMethod::Flat;
}
std::string ContentAddressMethod::render(HashType ht) const
@ -60,7 +61,7 @@ std::string ContentAddress::render() const
+ makeFileIngestionPrefix(method);
},
}, method.raw)
+ this->hash.to_string(Base32, true);
+ this->hash.to_string(HashFormat::Base32, true);
}
/**
@ -83,7 +84,7 @@ static std::pair<ContentAddressMethod, HashType> parseContentAddressMethodPrefix
if (!hashTypeRaw)
throw UsageError("content address hash must be in form '<algo>:<hash>', but found: %s", wholeInput);
HashType hashType = parseHashType(*hashTypeRaw);
return std::move(hashType);
return hashType;
};
// Switch on prefix

58
src/libstore/daemon.cc
View file

@ -45,9 +45,9 @@ struct TunnelLogger : public Logger
Sync<State> state_;
unsigned int clientVersion;
WorkerProto::Version clientVersion;
TunnelLogger(FdSink & to, unsigned int clientVersion)
TunnelLogger(FdSink & to, WorkerProto::Version clientVersion)
: to(to), clientVersion(clientVersion) { }
void enqueueMsg(const std::string & s)
@ -261,24 +261,18 @@ struct ClientSettings
}
};
static std::vector<DerivedPath> readDerivedPaths(Store & store, unsigned int clientVersion, WorkerProto::ReadConn conn)
{
std::vector<DerivedPath> reqs;
if (GET_PROTOCOL_MINOR(clientVersion) >= 30) {
reqs = WorkerProto::Serialise<std::vector<DerivedPath>>::read(store, conn);
} else {
for (auto & s : readStrings<Strings>(conn.from))
reqs.push_back(parsePathWithOutputs(store, s).toDerivedPath());
}
return reqs;
}
static void performOp(TunnelLogger * logger, ref<Store> store,
TrustedFlag trusted, RecursiveFlag recursive, unsigned int clientVersion,
TrustedFlag trusted, RecursiveFlag recursive, WorkerProto::Version clientVersion,
Source & from, BufferedSink & to, WorkerProto::Op op)
{
WorkerProto::ReadConn rconn { .from = from };
WorkerProto::WriteConn wconn { .to = to };
WorkerProto::ReadConn rconn {
.from = from,
.version = clientVersion,
};
WorkerProto::WriteConn wconn {
.to = to,
.version = clientVersion,
};
switch (op) {
@ -334,7 +328,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork();
auto hash = store->queryPathInfo(path)->narHash;
logger->stopWork();
to << hash.to_string(Base16, false);
to << hash.to_string(HashFormat::Base16, false);
break;
}
@ -428,7 +422,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}();
logger->stopWork();
pathInfo->write(to, *store, GET_PROTOCOL_MINOR(clientVersion));
WorkerProto::Serialise<ValidPathInfo>::write(*store, wconn, *pathInfo);
} else {
HashType hashAlgo;
std::string baseName;
@ -532,7 +526,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
case WorkerProto::Op::BuildPaths: {
auto drvs = readDerivedPaths(*store, clientVersion, rconn);
auto drvs = WorkerProto::Serialise<DerivedPaths>::read(*store, rconn);
BuildMode mode = bmNormal;
if (GET_PROTOCOL_MINOR(clientVersion) >= 15) {
mode = (BuildMode) readInt(from);
@ -557,7 +551,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
case WorkerProto::Op::BuildPathsWithResults: {
auto drvs = readDerivedPaths(*store, clientVersion, rconn);
auto drvs = WorkerProto::Serialise<DerivedPaths>::read(*store, rconn);
BuildMode mode = bmNormal;
mode = (BuildMode) readInt(from);
@ -641,16 +635,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
auto res = store->buildDerivation(drvPath, drv, buildMode);
logger->stopWork();
to << res.status << res.errorMsg;
if (GET_PROTOCOL_MINOR(clientVersion) >= 29) {
to << res.timesBuilt << res.isNonDeterministic << res.startTime << res.stopTime;
}
if (GET_PROTOCOL_MINOR(clientVersion) >= 28) {
DrvOutputs builtOutputs;
for (auto & [output, realisation] : res.builtOutputs)
builtOutputs.insert_or_assign(realisation.id, realisation);
WorkerProto::write(*store, wconn, builtOutputs);
}
WorkerProto::write(*store, wconn, res);
break;
}
@ -834,7 +819,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
if (info) {
if (GET_PROTOCOL_MINOR(clientVersion) >= 17)
to << 1;
info->write(to, *store, GET_PROTOCOL_MINOR(clientVersion), false);
WorkerProto::write(*store, wconn, static_cast<const UnkeyedValidPathInfo &>(*info));
} else {
assert(GET_PROTOCOL_MINOR(clientVersion) >= 17);
to << 0;
@ -932,7 +917,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
case WorkerProto::Op::QueryMissing: {
auto targets = readDerivedPaths(*store, clientVersion, rconn);
auto targets = WorkerProto::Serialise<DerivedPaths>::read(*store, rconn);
logger->startWork();
StorePathSet willBuild, willSubstitute, unknown;
uint64_t downloadSize, narSize;
@ -1017,7 +1002,7 @@ void processConnection(
if (magic != WORKER_MAGIC_1) throw Error("protocol mismatch");
to << WORKER_MAGIC_2 << PROTOCOL_VERSION;
to.flush();
unsigned int clientVersion = readInt(from);
WorkerProto::Version clientVersion = readInt(from);
if (clientVersion < 0x10a)
throw Error("the Nix client version is too old");
@ -1052,7 +1037,10 @@ void processConnection(
auto temp = trusted
? store->isTrustedClient()
: std::optional { NotTrusted };
WorkerProto::WriteConn wconn { .to = to };
WorkerProto::WriteConn wconn {
.to = to,
.version = clientVersion,
};
WorkerProto::write(*store, wconn, temp);
}

12
src/libstore/derivations.cc
View file

@ -542,7 +542,7 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs,
[&](const DerivationOutput::CAFixed & dof) {
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first)));
s += ','; printUnquotedString(s, dof.ca.printMethodAlgo());
s += ','; printUnquotedString(s, dof.ca.hash.to_string(Base16, false));
s += ','; printUnquotedString(s, dof.ca.hash.to_string(HashFormat::Base16, false));
},
[&](const DerivationOutput::CAFloating & dof) {
s += ','; printUnquotedString(s, "");
@ -775,7 +775,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
auto & dof = std::get<DerivationOutput::CAFixed>(i.second.raw);
auto hash = hashString(htSHA256, "fixed:out:"
+ dof.ca.printMethodAlgo() + ":"
+ dof.ca.hash.to_string(Base16, false) + ":"
+ dof.ca.hash.to_string(HashFormat::Base16, false) + ":"
+ store.printStorePath(dof.path(store, drv.name, i.first)));
outputHashes.insert_or_assign(i.first, std::move(hash));
}
@ -820,7 +820,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut
const auto h = get(res.hashes, outputName);
if (!h)
throw Error("no hash for output '%s' of derivation '%s'", outputName, drv.name);
inputs2[h->to_string(Base16, false)].value.insert(outputName);
inputs2[h->to_string(HashFormat::Base16, false)].value.insert(outputName);
}
}
@ -925,7 +925,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
[&](const DerivationOutput::CAFixed & dof) {
out << store.printStorePath(dof.path(store, drv.name, i.first))
<< dof.ca.printMethodAlgo()
<< dof.ca.hash.to_string(Base16, false);
<< dof.ca.hash.to_string(HashFormat::Base16, false);
},
[&](const DerivationOutput::CAFloating & dof) {
out << ""
@ -957,7 +957,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
std::string hashPlaceholder(const OutputNameView outputName)
{
// FIXME: memoize?
return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(Base32, false);
return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(HashFormat::Base32, false);
}
@ -1162,7 +1162,7 @@ nlohmann::json DerivationOutput::toJSON(
[&](const DerivationOutput::CAFixed & dof) {
res["path"] = store.printStorePath(dof.path(store, drvName, outputName));
res["hashAlgo"] = dof.ca.printMethodAlgo();
res["hash"] = dof.ca.hash.to_string(Base16, false);
res["hash"] = dof.ca.hash.to_string(HashFormat::Base16, false);
// FIXME print refs?
},
[&](const DerivationOutput::CAFloating & dof) {

4
src/libstore/downstream-placeholder.cc
View file

@ -5,7 +5,7 @@ namespace nix {
std::string DownstreamPlaceholder::render() const
{
return "/" + hash.to_string(Base32, false);
return "/" + hash.to_string(HashFormat::Base32, false);
}
@ -31,7 +31,7 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation(
xpSettings.require(Xp::DynamicDerivations);
auto compressed = compressHash(placeholder.hash, 20);
auto clearText = "nix-computed-output:"
+ compressed.to_string(Base32, false)
+ compressed.to_string(HashFormat::Base32, false)
+ ":" + std::string { outputName };
return DownstreamPlaceholder {
hashString(htSHA256, clearText)

2
src/libstore/export-import.cc
View file

@ -41,7 +41,7 @@ void Store::exportPath(const StorePath & path, Sink & sink)
Hash hash = hashSink.currentHash().first;
if (hash != info->narHash && info->narHash != Hash(info->narHash.type))
throw Error("hash of path '%s' has changed from '%s' to '%s'!",
printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true));
printStorePath(path), info->narHash.to_string(HashFormat::Base32, true), hash.to_string(HashFormat::Base32, true));
teeSink
<< exportMagic

2
src/libstore/gc.cc
View file

@ -43,7 +43,7 @@ static void makeSymlink(const Path & link, const Path & target)
void LocalStore::addIndirectRoot(const Path & path)
{
std::string hash = hashString(htSHA1, path).to_string(Base32, false);
std::string hash = hashString(htSHA1, path).to_string(HashFormat::Base32, false);
Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", stateDir, gcRootsDir, hash));
makeSymlink(realRoot, path);
}

31
src/libstore/globals.cc
View file

@ -24,6 +24,9 @@
#include "config-impl.hh"
#ifdef __APPLE__
#include <sys/sysctl.h>
#endif
namespace nix {
@ -154,6 +157,29 @@ unsigned int Settings::getDefaultCores()
return concurrency;
}
#if __APPLE__
static bool hasVirt() {
int hasVMM;
int hvSupport;
size_t size;
size = sizeof(hasVMM);
if (sysctlbyname("kern.hv_vmm_present", &hasVMM, &size, NULL, 0) == 0) {
if (hasVMM)
return false;
}
// whether the kernel and hardware supports virt
size = sizeof(hvSupport);
if (sysctlbyname("kern.hv_support", &hvSupport, &size, NULL, 0) == 0) {
return hvSupport == 1;
} else {
return false;
}
}
#endif
StringSet Settings::getDefaultSystemFeatures()
{
/* For backwards compatibility, accept some "features" that are
@ -170,6 +196,11 @@ StringSet Settings::getDefaultSystemFeatures()
features.insert("kvm");
#endif
#if __APPLE__
if (hasVirt())
features.insert("apple-virt");
#endif
return features;
}

6
src/libstore/globals.hh
View file

@ -714,9 +714,13 @@ public:
System features are user-defined, but Nix sets the following defaults:
- `apple-virt`
Included on Darwin if virtualization is available.
- `kvm`
Included by default if `/dev/kvm` is accessible.
Included on Linux if `/dev/kvm` is accessible.
- `nixos-test`, `benchmark`, `big-parallel`

21
src/libstore/legacy-ssh-store.cc
View file

@ -45,7 +45,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
std::unique_ptr<SSHMaster::Connection> sshConn;
FdSink to;
FdSource from;
int remoteVersion;
ServeProto::Version remoteVersion;
bool good = true;
/**
@ -60,6 +60,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
{
return ServeProto::ReadConn {
.from = from,
.version = remoteVersion,
};
}
@ -75,6 +76,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
{
return ServeProto::WriteConn {
.to = to,
.version = remoteVersion,
};
}
};
@ -209,7 +211,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor
<< ServeProto::Command::AddToStoreNar
<< printStorePath(info.path)
<< (info.deriver ? printStorePath(*info.deriver) : "")
<< info.narHash.to_string(Base16, false);
<< info.narHash.to_string(HashFormat::Base16, false);
ServeProto::write(*this, *conn, info.references);
conn->to
<< info.registrationTime
@ -317,20 +319,7 @@ public:
conn->to.flush();
BuildResult status;
status.status = (BuildResult::Status) readInt(conn->from);
conn->from >> status.errorMsg;
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3)
conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime;
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) {
auto builtOutputs = ServeProto::Serialise<DrvOutputs>::read(*this, *conn);
for (auto && [output, realisation] : builtOutputs)
status.builtOutputs.insert_or_assign(
std::move(output.outputName),
std::move(realisation));
}
return status;
return ServeProto::Serialise<BuildResult>::read(*this, *conn);
}
void buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override

14
src/libstore/local-store.cc
View file

@ -826,7 +826,7 @@ uint64_t LocalStore::addValidPath(State & state,
state.stmts->RegisterValidPath.use()
(printStorePath(info.path))
(info.narHash.to_string(Base16, true))
(info.narHash.to_string(HashFormat::Base16, true))
(info.registrationTime == 0 ? time(0) : info.registrationTime)
(info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver)
(info.narSize, info.narSize != 0)
@ -933,7 +933,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
{
state.stmts->UpdatePathInfo.use()
(info.narSize, info.narSize != 0)
(info.narHash.to_string(Base16, true))
(info.narHash.to_string(HashFormat::Base16, true))
(info.ultimate ? 1 : 0, info.ultimate)
(concatStringsSep(" ", info.sigs), !info.sigs.empty())
(renderContentAddress(info.ca), (bool) info.ca)
@ -1236,7 +1236,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
if (hashResult.first != info.narHash)
throw Error("hash mismatch importing path '%s';\n specified: %s\n got: %s",
printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true));
printStorePath(info.path), info.narHash.to_string(HashFormat::Base32, true), hashResult.first.to_string(HashFormat::Base32, true));
if (hashResult.second != info.narSize)
throw Error("size mismatch importing path '%s';\n specified: %s\n got: %s",
@ -1252,8 +1252,8 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
if (specified.hash != actualHash.hash) {
throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s",
printStorePath(info.path),
specified.hash.to_string(Base32, true),
actualHash.hash.to_string(Base32, true));
specified.hash.to_string(HashFormat::Base32, true),
actualHash.hash.to_string(HashFormat::Base32, true));
}
}
@ -1545,7 +1545,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
for (auto & link : readDirectory(linksDir)) {
printMsg(lvlTalkative, "checking contents of '%s'", link.name);
Path linkPath = linksDir + "/" + link.name;
std::string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false);
std::string hash = hashPath(htSHA256, linkPath).first.to_string(HashFormat::Base32, false);
if (hash != link.name) {
printError("link '%s' was modified! expected hash '%s', got '%s'",
linkPath, link.name, hash);
@ -1578,7 +1578,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
if (info->narHash != nullHash && info->narHash != current.first) {
printError("path '%s' was modified! expected hash '%s', got '%s'",
printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true));
printStorePath(i), info->narHash.to_string(HashFormat::Base32, true), current.first.to_string(HashFormat::Base32, true));
if (repair) repairPath(i); else errors = true;
} else {

60
src/libstore/lock.cc
View file

@ -7,6 +7,31 @@
namespace nix {
#if __linux__
static std::vector<gid_t> get_group_list(const char *username, gid_t group_id)
{
std::vector<gid_t> gids;
gids.resize(32); // Initial guess
auto getgroupl_failed {[&] {
int ngroups = gids.size();
int err = getgrouplist(username, group_id, gids.data(), &ngroups);
gids.resize(ngroups);
return err == -1;
}};
// The first error means that the vector was not big enough.
// If it happens again, there is some different problem.
if (getgroupl_failed() && getgroupl_failed()) {
throw SysError("failed to get list of supplementary groups for '%s'", username);
}
return gids;
}
#endif
struct SimpleUserLock : UserLock
{
AutoCloseFD fdUserLock;
@ -67,37 +92,14 @@ struct SimpleUserLock : UserLock
throw Error("the Nix user should not be a member of '%s'", settings.buildUsersGroup);
#if __linux__
/* Get the list of supplementary groups of this build
user. This is usually either empty or contains a
group such as "kvm". */
int ngroups = 32; // arbitrary initial guess
std::vector<gid_t> gids;
gids.resize(ngroups);
int err = getgrouplist(
pw->pw_name, pw->pw_gid,
gids.data(),
&ngroups);
/* Our initial size of 32 wasn't sufficient, the
correct size has been stored in ngroups, so we try
again. */
if (err == -1) {
gids.resize(ngroups);
err = getgrouplist(
pw->pw_name, pw->pw_gid,
gids.data(),
&ngroups);
}
// If it failed once more, then something must be broken.
if (err == -1)
throw Error("failed to get list of supplementary groups for '%s'", pw->pw_name);
/* Get the list of supplementary groups of this user. This is
* usually either empty or contains a group such as "kvm". */
// Finally, trim back the GID list to its real size.
for (auto i = 0; i < ngroups; i++)
if (gids[i] != lock->gid)
lock->supplementaryGIDs.push_back(gids[i]);
for (auto gid : get_group_list(pw->pw_name, pw->pw_gid)) {
if (gid != lock->gid)
lock->supplementaryGIDs.push_back(gid);
}
#endif
return lock;

4
src/libstore/nar-info-disk-cache.cc
View file

@ -332,9 +332,9 @@ public:
(std::string(info->path.name()))
(narInfo ? narInfo->url : "", narInfo != 0)
(narInfo ? narInfo->compression : "", narInfo != 0)
(narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(Base32, true) : "", narInfo && narInfo->fileHash)
(narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(HashFormat::Base32, true) : "", narInfo && narInfo->fileHash)
(narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize)
(info->narHash.to_string(Base32, true))
(info->narHash.to_string(HashFormat::Base32, true))
(info->narSize)
(concatStringsSep(" ", info->shortRefs()))
(info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver)

4
src/libstore/nar-info.cc
View file

@ -105,10 +105,10 @@ std::string NarInfo::to_string(const Store & store) const
assert(compression != "");
res += "Compression: " + compression + "\n";
assert(fileHash && fileHash->type == htSHA256);
res += "FileHash: " + fileHash->to_string(Base32, true) + "\n";
res += "FileHash: " + fileHash->to_string(HashFormat::Base32, true) + "\n";
res += "FileSize: " + std::to_string(fileSize) + "\n";
assert(narHash.type == htSHA256);
res += "NarHash: " + narHash.to_string(Base32, true) + "\n";
res += "NarHash: " + narHash.to_string(HashFormat::Base32, true) + "\n";
res += "NarSize: " + std::to_string(narSize) + "\n";
res += "References: " + concatStringsSep(" ", shortRefs()) + "\n";

4
src/libstore/optimise-store.cc
View file

@ -146,10 +146,10 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
contents of the symlink (i.e. the result of readlink()), not
the contents of the target (which may not even exist). */
Hash hash = hashPath(htSHA256, path).first;
debug("'%1%' has hash '%2%'", path, hash.to_string(Base32, true));
debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Base32, true));
/* Check if this is a known hash. */
Path linkPath = linksDir + "/" + hash.to_string(Base32, false);
Path linkPath = linksDir + "/" + hash.to_string(HashFormat::Base32, false);
/* Maybe delete the link, if it has been corrupted. */
if (pathExists(linkPath)) {

2
src/libstore/outputs-spec.cc
View file

@ -63,7 +63,7 @@ std::optional<std::pair<std::string_view, ExtendedOutputsSpec>> ExtendedOutputsS
auto specOpt = OutputsSpec::parseOpt(s.substr(found + 1));
if (!specOpt)
return std::nullopt;
return std::pair { s.substr(0, found), ExtendedOutputsSpec::Explicit { *std::move(specOpt) } };
return std::pair { s.substr(0, found), ExtendedOutputsSpec::Explicit { std::move(*specOpt) } };
}

73
src/libstore/path-info.cc
View file

@ -1,10 +1,27 @@
#include "path-info.hh"
#include "worker-protocol.hh"
#include "worker-protocol-impl.hh"
#include "store-api.hh"
namespace nix {
GENERATE_CMP_EXT(
,
UnkeyedValidPathInfo,
me->deriver,
me->narHash,
me->references,
me->registrationTime,
me->narSize,
//me->id,
me->ultimate,
me->sigs,
me->ca);
GENERATE_CMP_EXT(
,
ValidPathInfo,
me->path,
static_cast<const UnkeyedValidPathInfo &>(*me));
std::string ValidPathInfo::fingerprint(const Store & store) const
{
if (narSize == 0)
@ -12,7 +29,7 @@ std::string ValidPathInfo::fingerprint(const Store & store) const
store.printStorePath(path));
return
"1;" + store.printStorePath(path) + ";"
+ narHash.to_string(Base32, true) + ";"
+ narHash.to_string(HashFormat::Base32, true) + ";"
+ std::to_string(narSize) + ";"
+ concatStringsSep(",", store.printStorePathSet(references));
}
@ -99,14 +116,13 @@ Strings ValidPathInfo::shortRefs() const
return refs;
}
ValidPathInfo::ValidPathInfo(
const Store & store,
std::string_view name,
ContentAddressWithReferences && ca,
Hash narHash)
: path(store.makeFixedOutputPathFromCA(name, ca))
, narHash(narHash)
: UnkeyedValidPathInfo(narHash)
, path(store.makeFixedOutputPathFromCA(name, ca))
{
std::visit(overloaded {
[this](TextInfo && ti) {
@ -128,49 +144,4 @@ ValidPathInfo::ValidPathInfo(
}, std::move(ca).raw);
}
ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format)
{
return read(source, store, format, store.parseStorePath(readString(source)));
}
ValidPathInfo ValidPathInfo::read(Source & source, const Store & store, unsigned int format, StorePath && path)
{
auto deriver = readString(source);
auto narHash = Hash::parseAny(readString(source), htSHA256);
ValidPathInfo info(path, narHash);
if (deriver != "") info.deriver = store.parseStorePath(deriver);
info.references = WorkerProto::Serialise<StorePathSet>::read(store,
WorkerProto::ReadConn { .from = source });
source >> info.registrationTime >> info.narSize;
if (format >= 16) {
source >> info.ultimate;
info.sigs = readStrings<StringSet>(source);
info.ca = ContentAddress::parseOpt(readString(source));
}
return info;
}
void ValidPathInfo::write(
Sink & sink,
const Store & store,
unsigned int format,
bool includePath) const
{
if (includePath)
sink << store.printStorePath(path);
sink << (deriver ? store.printStorePath(*deriver) : "")
<< narHash.to_string(Base16, false);
WorkerProto::write(store,
WorkerProto::WriteConn { .to = sink },
references);
sink << registrationTime << narSize;
if (format >= 16) {
sink << ultimate
<< sigs
<< renderContentAddress(ca);
}
}
}

48
src/libstore/path-info.hh
View file

@ -29,12 +29,11 @@ struct SubstitutablePathInfo
uint64_t narSize;
};
typedef std::map<StorePath, SubstitutablePathInfo> SubstitutablePathInfos;
using SubstitutablePathInfos = std::map<StorePath, SubstitutablePathInfo>;
struct ValidPathInfo
struct UnkeyedValidPathInfo
{
StorePath path;
std::optional<StorePath> deriver;
/**
* \todo document this
@ -43,7 +42,7 @@ struct ValidPathInfo
StorePathSet references;
time_t registrationTime = 0;
uint64_t narSize = 0; // 0 = unknown
uint64_t id; // internal use only
uint64_t id = 0; // internal use only
/**
* Whether the path is ultimately trusted, that is, it's a
@ -72,13 +71,19 @@ struct ValidPathInfo
*/
std::optional<ContentAddress> ca;
bool operator == (const ValidPathInfo & i) const
{
return
path == i.path
&& narHash == i.narHash
&& references == i.references;
}
UnkeyedValidPathInfo(const UnkeyedValidPathInfo & other) = default;
UnkeyedValidPathInfo(Hash narHash) : narHash(narHash) { };
DECLARE_CMP(UnkeyedValidPathInfo);
virtual ~UnkeyedValidPathInfo() { }
};
struct ValidPathInfo : UnkeyedValidPathInfo {
StorePath path;
DECLARE_CMP(ValidPathInfo);
/**
* Return a fingerprint of the store path to be used in binary
@ -92,11 +97,11 @@ struct ValidPathInfo
void sign(const Store & store, const SecretKey & secretKey);
/**
* @return The `ContentAddressWithReferences` that determines the
* store path for a content-addressed store object, `std::nullopt`
* for an input-addressed store object.
*/
/**
* @return The `ContentAddressWithReferences` that determines the
* store path for a content-addressed store object, `std::nullopt`
* for an input-addressed store object.
*/
std::optional<ContentAddressWithReferences> contentAddressWithReferences() const;
/**
@ -122,20 +127,15 @@ struct ValidPathInfo
ValidPathInfo(const ValidPathInfo & other) = default;
ValidPathInfo(StorePath && path, Hash narHash) : path(std::move(path)), narHash(narHash) { };
ValidPathInfo(const StorePath & path, Hash narHash) : path(path), narHash(narHash) { };
ValidPathInfo(StorePath && path, UnkeyedValidPathInfo info) : UnkeyedValidPathInfo(info), path(std::move(path)) { };
ValidPathInfo(const StorePath & path, UnkeyedValidPathInfo info) : UnkeyedValidPathInfo(info), path(path) { };
ValidPathInfo(const Store & store,
std::string_view name, ContentAddressWithReferences && ca, Hash narHash);
virtual ~ValidPathInfo() { }
static ValidPathInfo read(Source & source, const Store & store, unsigned int format);
static ValidPathInfo read(Source & source, const Store & store, unsigned int format, StorePath && path);
void write(Sink & sink, const Store & store, unsigned int format, bool includePath = true) const;
};
typedef std::map<StorePath, ValidPathInfo> ValidPathInfos;
using ValidPathInfos = std::map<StorePath, ValidPathInfo>;
}

2
src/libstore/path.cc
View file

@ -35,7 +35,7 @@ StorePath::StorePath(std::string_view _baseName)
}
StorePath::StorePath(const Hash & hash, std::string_view _name)
: baseName((hash.to_string(Base32, false) + "-").append(std::string(_name)))
: baseName((hash.to_string(HashFormat::Base32, false) + "-").append(std::string(_name)))
{
checkName(baseName, name());
}

2
src/libstore/realisation.hh
View file

@ -39,7 +39,7 @@ struct DrvOutput {
std::string to_string() const;
std::string strHash() const
{ return drvHash.to_string(Base16, true); }
{ return drvHash.to_string(HashFormat::Base16, true); }
static DrvOutput parse(const std::string &);

4
src/libstore/remote-store-connection.hh
View file

@ -30,7 +30,7 @@ struct RemoteStore::Connection
* sides support. (If the maximum doesn't exist, we would fail to
* establish a connection and produce a value of this type.)
*/
unsigned int daemonVersion;
WorkerProto::Version daemonVersion;
/**
* Whether the remote side trusts us or not.
@ -70,6 +70,7 @@ struct RemoteStore::Connection
{
return WorkerProto::ReadConn {
.from = from,
.version = daemonVersion,
};
}
@ -85,6 +86,7 @@ struct RemoteStore::Connection
{
return WorkerProto::WriteConn {
.to = to,
.version = daemonVersion,
};
}

62
src/libstore/remote-store.cc
View file

@ -332,7 +332,8 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path,
if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path));
}
info = std::make_shared<ValidPathInfo>(
ValidPathInfo::read(conn->from, *this, GET_PROTOCOL_MINOR(conn->daemonVersion), StorePath{path}));
StorePath{path},
WorkerProto::Serialise<UnkeyedValidPathInfo>::read(*this, *conn));
}
callback(std::move(info));
} catch (...) { callback.rethrow(); }
@ -445,7 +446,7 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
}
return make_ref<ValidPathInfo>(
ValidPathInfo::read(conn->from, *this, GET_PROTOCOL_MINOR(conn->daemonVersion)));
WorkerProto::Serialise<ValidPathInfo>::read(*this, *conn));
}
else {
if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25");
@ -541,7 +542,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,
conn->to << WorkerProto::Op::AddToStoreNar
<< printStorePath(info.path)
<< (info.deriver ? printStorePath(*info.deriver) : "")
<< info.narHash.to_string(Base16, false);
<< info.narHash.to_string(HashFormat::Base16, false);
WorkerProto::write(*this, *conn, info.references);
conn->to << info.registrationTime << info.narSize
<< info.ultimate << info.sigs << renderContentAddress(info.ca)
@ -570,7 +571,12 @@ void RemoteStore::addMultipleToStore(
auto source = sinkToSource([&](Sink & sink) {
sink << pathsToCopy.size();
for (auto & [pathInfo, pathSource] : pathsToCopy) {
pathInfo.write(sink, *this, 16);
WorkerProto::Serialise<ValidPathInfo>::write(*this,
WorkerProto::WriteConn {
.to = sink,
.version = 16,
},
pathInfo);
pathSource->drainInto(sink);
}
});
@ -655,33 +661,6 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id,
} catch (...) { return callback.rethrow(); }
}
static void writeDerivedPaths(RemoteStore & store, RemoteStore::Connection & conn, const std::vector<DerivedPath> & reqs)
{
if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 30) {
WorkerProto::write(store, conn, reqs);
} else {
Strings ss;
for (auto & p : reqs) {
auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p);
std::visit(overloaded {
[&](const StorePathWithOutputs & s) {
ss.push_back(s.to_string(store));
},
[&](const StorePath & drvPath) {
throw Error("trying to request '%s', but daemon protocol %d.%d is too old (< 1.29) to request a derivation file",
store.printStorePath(drvPath),
GET_PROTOCOL_MAJOR(conn.daemonVersion),
GET_PROTOCOL_MINOR(conn.daemonVersion));
},
[&](std::monostate) {
throw Error("wanted to build a derivation that is itself a build product, but the legacy 'ssh://' protocol doesn't support that. Try using 'ssh-ng://'");
},
}, sOrDrvPath);
}
conn.to << ss;
}
}
void RemoteStore::copyDrvsFromEvalStore(
const std::vector<DerivedPath> & paths,
std::shared_ptr<Store> evalStore)
@ -711,7 +690,7 @@ void RemoteStore::buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMod
auto conn(getConnection());
conn->to << WorkerProto::Op::BuildPaths;
assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13);
writeDerivedPaths(*this, *conn, drvPaths);
WorkerProto::write(*this, *conn, drvPaths);
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15)
conn->to << buildMode;
else
@ -735,7 +714,7 @@ std::vector<KeyedBuildResult> RemoteStore::buildPathsWithResults(
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 34) {
conn->to << WorkerProto::Op::BuildPathsWithResults;
writeDerivedPaths(*this, *conn, paths);
WorkerProto::write(*this, *conn, paths);
conn->to << buildMode;
conn.processStderr();
return WorkerProto::Serialise<std::vector<KeyedBuildResult>>::read(*this, *conn);
@ -815,20 +794,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD
writeDerivation(conn->to, *this, drv);
conn->to << buildMode;
conn.processStderr();
BuildResult res;
res.status = (BuildResult::Status) readInt(conn->from);
conn->from >> res.errorMsg;
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) {
conn->from >> res.timesBuilt >> res.isNonDeterministic >> res.startTime >> res.stopTime;
}
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 28) {
auto builtOutputs = WorkerProto::Serialise<DrvOutputs>::read(*this, *conn);
for (auto && [output, realisation] : builtOutputs)
res.builtOutputs.insert_or_assign(
std::move(output.outputName),
std::move(realisation));
}
return res;
return WorkerProto::Serialise<BuildResult>::read(*this, *conn);
}
@ -929,7 +895,7 @@ void RemoteStore::queryMissing(const std::vector<DerivedPath> & targets,
// to prevent a deadlock.
goto fallback;
conn->to << WorkerProto::Op::QueryMissing;
writeDerivedPaths(*this, *conn, targets);
WorkerProto::write(*this, *conn, targets);
conn.processStderr();
willBuild = WorkerProto::Serialise<StorePathSet>::read(*this, *conn);
willSubstitute = WorkerProto::Serialise<StorePathSet>::read(*this, *conn);

100
src/libstore/s3-binary-cache-store.md
View file

@ -2,7 +2,103 @@ R"(
**Store URL format**: `s3://`*bucket-name*
This store allows reading and writing a binary cache stored in an AWS
S3 bucket.
This store allows reading and writing a binary cache stored in an AWS S3 (or S3-compatible service) bucket.
This store shares many idioms with the [HTTP Binary Cache Store](#http-binary-cache-store).
For AWS S3, the binary cache URL for a bucket named `example-nix-cache` will be exactly <s3://example-nix-cache>.
For S3 compatible binary caches, consult that cache's documentation.
### Anonymous reads to your S3-compatible binary cache
> If your binary cache is publicly accessible and does not require authentication,
> it is simplest to use the [HTTP Binary Cache Store] rather than S3 Binary Cache Store with
> <https://example-nix-cache.s3.amazonaws.com> instead of <s3://example-nix-cache>.
Your bucket will need a
[bucket policy](https://docs.aws.amazon.com/AmazonS3/v1/userguide/bucket-policies.html)
like the following to be accessible:
```json
{
"Id": "DirectReads",
"Version": "2012-10-17",
"Statement": [
{
"Sid": "AllowDirectReads",
"Action": [
"s3:GetObject",
"s3:GetBucketLocation"
],
"Effect": "Allow",
"Resource": [
"arn:aws:s3:::example-nix-cache",
"arn:aws:s3:::example-nix-cache/*"
],
"Principal": "*"
}
]
}
```
### Authentication
Nix will use the
[default credential provider chain](https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html)
for authenticating requests to Amazon S3.
Note that this means Nix will read environment variables and files with different idioms than with Nix's own settings, as implemented by the AWS SDK.
Consult the documentation linked above for further details.
### Authenticated reads to your S3 binary cache
Your bucket will need a bucket policy allowing the desired users to perform the `s3:GetObject` and `s3:GetBucketLocation` action on all objects in the bucket.
The [anonymous policy given above](#anonymous-reads-to-your-s3-compatible-binary-cache) can be updated to have a restricted `Principal` to support this.
### Authenticated writes to your S3-compatible binary cache
Your account will need an IAM policy to support uploading to the bucket:
```json
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "UploadToCache",
"Effect": "Allow",
"Action": [
"s3:AbortMultipartUpload",
"s3:GetBucketLocation",
"s3:GetObject",
"s3:ListBucket",
"s3:ListBucketMultipartUploads",
"s3:ListMultipartUploadParts",
"s3:PutObject"
],
"Resource": [
"arn:aws:s3:::example-nix-cache",
"arn:aws:s3:::example-nix-cache/*"
]
}
]
}
```
### Examples
With bucket policies and authentication set up as described above, uploading works via [`nix copy`](@docroot@/command-ref/new-cli/nix3-copy.md) (experimental).
- To upload with a specific credential profile for Amazon S3:
```console
$ nix copy nixpkgs.hello \
--to 's3://example-nix-cache?profile=cache-upload&region=eu-west-2'
```
- To upload to an S3-compatible binary cache:
```console
$ nix copy nixpkgs.hello --to \
's3://example-nix-cache?profile=cache-upload&scheme=https&endpoint=minio.example.com'
```
)"

43
src/libstore/serve-protocol.cc
View file

@ -2,6 +2,7 @@
#include "util.hh"
#include "path-with-outputs.hh"
#include "store-api.hh"
#include "build-result.hh"
#include "serve-protocol.hh"
#include "serve-protocol-impl.hh"
#include "archive.hh"
@ -12,4 +13,46 @@ namespace nix {
/* protocol-specific definitions */
BuildResult ServeProto::Serialise<BuildResult>::read(const Store & store, ServeProto::ReadConn conn)
{
BuildResult status;
status.status = (BuildResult::Status) readInt(conn.from);
conn.from >> status.errorMsg;
if (GET_PROTOCOL_MINOR(conn.version) >= 3)
conn.from
>> status.timesBuilt
>> status.isNonDeterministic
>> status.startTime
>> status.stopTime;
if (GET_PROTOCOL_MINOR(conn.version) >= 6) {
auto builtOutputs = ServeProto::Serialise<DrvOutputs>::read(store, conn);
for (auto && [output, realisation] : builtOutputs)
status.builtOutputs.insert_or_assign(
std::move(output.outputName),
std::move(realisation));
}
return status;
}
void ServeProto::Serialise<BuildResult>::write(const Store & store, ServeProto::WriteConn conn, const BuildResult & status)
{
conn.to
<< status.status
<< status.errorMsg;
if (GET_PROTOCOL_MINOR(conn.version) >= 3)
conn.to
<< status.timesBuilt
<< status.isNonDeterministic
<< status.startTime
<< status.stopTime;
if (GET_PROTOCOL_MINOR(conn.version) >= 6) {
DrvOutputs builtOutputs;
for (auto & [output, realisation] : status.builtOutputs)
builtOutputs.insert_or_assign(realisation.id, realisation);
ServeProto::write(store, conn, builtOutputs);
}
}
}

21
src/libstore/serve-protocol.hh
View file

@ -16,6 +16,9 @@ namespace nix {
class Store;
struct Source;
// items being serialised
struct BuildResult;
/**
* The "serve protocol", used by ssh:// stores.
@ -30,26 +33,29 @@ struct ServeProto
*/
enum struct Command : uint64_t;
/**
* Version type for the protocol.
*
* @todo Convert to struct with separate major vs minor fields.
*/
using Version = unsigned int;
/**
* A unidirectional read connection, to be used by the read half of the
* canonical serializers below.
*
* This currently is just a `Source &`, but more fields will be added
* later.
*/
struct ReadConn {
Source & from;
Version version;
};
/**
* A unidirectional write connection, to be used by the write half of the
* canonical serializers below.
*
* This currently is just a `Sink &`, but more fields will be added
* later.
*/
struct WriteConn {
Sink & to;
Version version;
};
/**
@ -133,6 +139,9 @@ inline std::ostream & operator << (std::ostream & s, ServeProto::Command op)
static void write(const Store & store, ServeProto::WriteConn conn, const T & t); \
};
template<>
DECLARE_SERVE_SERIALISER(BuildResult);
template<typename T>
DECLARE_SERVE_SERIALISER(std::vector<T>);
template<typename T>

37
src/libstore/store-api.cc
View file

@ -11,6 +11,9 @@
#include "archive.hh"
#include "callback.hh"
#include "remote-store.hh"
// FIXME this should not be here, see TODO below on
// `addMultipleToStore`.
#include "worker-protocol.hh"
#include <nlohmann/json.hpp>
#include <regex>
@ -154,7 +157,7 @@ StorePath Store::makeStorePath(std::string_view type,
StorePath Store::makeStorePath(std::string_view type,
const Hash & hash, std::string_view name) const
{
return makeStorePath(type, hash.to_string(Base16, true), name);
return makeStorePath(type, hash.to_string(HashFormat::Base16, true), name);
}
@ -192,7 +195,7 @@ StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInf
hashString(htSHA256,
"fixed:out:"
+ makeFileIngestionPrefix(info.method)
+ info.hash.to_string(Base16, true) + ":"),
+ info.hash.to_string(HashFormat::Base16, true) + ":"),
name);
}
}
@ -225,12 +228,16 @@ StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentA
}
std::pair<StorePath, Hash> Store::computeStorePathForPath(std::string_view name,
const Path & srcPath, FileIngestionMethod method, HashType hashAlgo, PathFilter & filter) const
std::pair<StorePath, Hash> Store::computeStorePathFromDump(
Source & dump,
std::string_view name,
FileIngestionMethod method,
HashType hashAlgo,
const StorePathSet & references) const
{
Hash h = method == FileIngestionMethod::Recursive
? hashPath(hashAlgo, srcPath, filter).first
: hashFile(hashAlgo, srcPath);
HashSink sink(hashAlgo);
dump.drainInto(sink);
auto h = sink.finish().first;
FixedOutputInfo caInfo {
.method = method,
.hash = h,
@ -357,7 +364,13 @@ void Store::addMultipleToStore(
{
auto expected = readNum<uint64_t>(source);
for (uint64_t i = 0; i < expected; ++i) {
auto info = ValidPathInfo::read(source, *this, 16);
// FIXME we should not be using the worker protocol here, let
// alone the worker protocol with a hard-coded version!
auto info = WorkerProto::Serialise<ValidPathInfo>::read(*this,
WorkerProto::ReadConn {
.from = source,
.version = 16,
});
info.ultimate = false;
addToStore(info, source, repair, checkSigs);
}
@ -884,7 +897,7 @@ std::string Store::makeValidityRegistration(const StorePathSet & paths,
auto info = queryPathInfo(i);
if (showHash) {
s += info->narHash.to_string(Base16, false) + "\n";
s += info->narHash.to_string(HashFormat::Base16, false) + "\n";
s += fmt("%1%\n", info->narSize);
}
@ -938,7 +951,7 @@ StorePathSet Store::exportReferences(const StorePathSet & storePaths, const Stor
json Store::pathInfoToJSON(const StorePathSet & storePaths,
bool includeImpureInfo, bool showClosureSize,
Base hashBase,
HashFormat hashFormat,
AllowInvalidFlag allowInvalid)
{
json::array_t jsonList = json::array();
@ -951,7 +964,7 @@ json Store::pathInfoToJSON(const StorePathSet & storePaths,
jsonPath["path"] = printStorePath(info->path);
jsonPath["valid"] = true;
jsonPath["narHash"] = info->narHash.to_string(hashBase, true);
jsonPath["narHash"] = info->narHash.to_string(hashFormat, true);
jsonPath["narSize"] = info->narSize;
{
@ -993,7 +1006,7 @@ json Store::pathInfoToJSON(const StorePathSet & storePaths,
if (!narInfo->url.empty())
jsonPath["url"] = narInfo->url;
if (narInfo->fileHash)
jsonPath["downloadHash"] = narInfo->fileHash->to_string(hashBase, true);
jsonPath["downloadHash"] = narInfo->fileHash->to_string(hashFormat, true);
if (narInfo->fileSize)
jsonPath["downloadSize"] = narInfo->fileSize;
if (showClosureSize)

17
src/libstore/store-api.hh
View file

@ -292,14 +292,15 @@ public:
StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const;
/**
* Preparatory part of addToStore().
*
* @return the store path to which srcPath is to be copied
* and the cryptographic hash of the contents of srcPath.
* Read-only variant of addToStoreFromDump(). It returns the store
* path to which a NAR or flat file would be written.
*/
std::pair<StorePath, Hash> computeStorePathForPath(std::string_view name,
const Path & srcPath, FileIngestionMethod method = FileIngestionMethod::Recursive,
HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const;
std::pair<StorePath, Hash> computeStorePathFromDump(
Source & dump,
std::string_view name,
FileIngestionMethod method = FileIngestionMethod::Recursive,
HashType hashAlgo = htSHA256,
const StorePathSet & references = {}) const;
/**
* Preparatory part of addTextToStore().
@ -676,7 +677,7 @@ public:
*/
nlohmann::json pathInfoToJSON(const StorePathSet & storePaths,
bool includeImpureInfo, bool showClosureSize,
Base hashBase = Base32,
HashFormat hashFormat = HashFormat::Base32,
AllowInvalidFlag allowInvalid = DisallowInvalid);
/**

5
src/libstore/tests/characterization.hh
View file

@ -20,4 +20,9 @@ static bool testAccept() {
return getEnv("_NIX_TEST_ACCEPT") == "1";
}
constexpr std::string_view cannotReadGoldenMaster =
"Cannot read golden master because another test is also updating it";
constexpr std::string_view updatingGoldenMaster =
"Updating golden master";
}

73
src/libstore/tests/common-protocol.cc
View file

@ -13,10 +13,71 @@ namespace nix {
const char commonProtoDir[] = "common-protocol";
using CommonProtoTest = ProtoTest<CommonProto, commonProtoDir>;
class CommonProtoTest : public ProtoTest<CommonProto, commonProtoDir>
{
public:
/**
* Golden test for `T` reading
*/
template<typename T>
void readTest(PathView testStem, T value)
{
if (testAccept())
{
GTEST_SKIP() << cannotReadGoldenMaster;
}
else
{
auto encoded = readFile(goldenMaster(testStem));
T got = ({
StringSource from { encoded };
CommonProto::Serialise<T>::read(
*store,
CommonProto::ReadConn { .from = from });
});
ASSERT_EQ(got, value);
}
}
/**
* Golden test for `T` write
*/
template<typename T>
void writeTest(PathView testStem, const T & value)
{
auto file = goldenMaster(testStem);
StringSink to;
CommonProto::write(
*store,
CommonProto::WriteConn { .to = to },
value);
if (testAccept())
{
createDirs(dirOf(file));
writeFile(file, to.s);
GTEST_SKIP() << updatingGoldenMaster;
}
else
{
auto expected = readFile(file);
ASSERT_EQ(to.s, expected);
}
}
};
#define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \
TEST_F(CommonProtoTest, NAME ## _read) { \
readTest(STEM, VALUE); \
} \
TEST_F(CommonProtoTest, NAME ## _write) { \
writeTest(STEM, VALUE); \
}
CHARACTERIZATION_TEST(
CommonProtoTest,
string,
"string",
(std::tuple<std::string, std::string, std::string, std::string, std::string> {
@ -28,7 +89,6 @@ CHARACTERIZATION_TEST(
}))
CHARACTERIZATION_TEST(
CommonProtoTest,
storePath,
"store-path",
(std::tuple<StorePath, StorePath> {
@ -37,7 +97,6 @@ CHARACTERIZATION_TEST(
}))
CHARACTERIZATION_TEST(
CommonProtoTest,
contentAddress,
"content-address",
(std::tuple<ContentAddress, ContentAddress, ContentAddress> {
@ -56,7 +115,6 @@ CHARACTERIZATION_TEST(
}))
CHARACTERIZATION_TEST(
CommonProtoTest,
drvOutput,
"drv-output",
(std::tuple<DrvOutput, DrvOutput> {
@ -71,7 +129,6 @@ CHARACTERIZATION_TEST(
}))
CHARACTERIZATION_TEST(
CommonProtoTest,
realisation,
"realisation",
(std::tuple<Realisation, Realisation> {
@ -103,7 +160,6 @@ CHARACTERIZATION_TEST(
}))
CHARACTERIZATION_TEST(
CommonProtoTest,
vector,
"vector",
(std::tuple<std::vector<std::string>, std::vector<std::string>, std::vector<std::string>, std::vector<std::vector<std::string>>> {
@ -114,7 +170,6 @@ CHARACTERIZATION_TEST(
}))
CHARACTERIZATION_TEST(
CommonProtoTest,
set,
"set",
(std::tuple<std::set<std::string>, std::set<std::string>, std::set<std::string>, std::set<std::set<std::string>>> {
@ -125,7 +180,6 @@ CHARACTERIZATION_TEST(
}))
CHARACTERIZATION_TEST(
CommonProtoTest,
optionalStorePath,
"optional-store-path",
(std::tuple<std::optional<StorePath>, std::optional<StorePath>> {
@ -136,7 +190,6 @@ CHARACTERIZATION_TEST(
}))
CHARACTERIZATION_TEST(
CommonProtoTest,
optionalContentAddress,
"optional-content-address",
(std::tuple<std::optional<ContentAddress>, std::optional<ContentAddress>> {

273
src/libstore/tests/derivation.cc
View file

@ -5,9 +5,12 @@
#include "derivations.hh"
#include "tests/libstore.hh"
#include "tests/characterization.hh"
namespace nix {
using nlohmann::json;
class DerivationTest : public LibStoreTest
{
public:
@ -16,6 +19,12 @@ public:
* to worry about race conditions if the tests run concurrently.
*/
ExperimentalFeatureSettings mockXpSettings;
Path unitTestData = getUnitTestData() + "/libstore/derivation";
Path goldenMaster(std::string_view testStem) {
return unitTestData + "/" + testStem;
}
};
class CaDerivationTest : public DerivationTest
@ -46,7 +55,7 @@ TEST_F(DerivationTest, BadATerm_version) {
ASSERT_THROW(
parseDerivation(
*store,
R"(DrvWithVersion("invalid-version",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))",
readFile(goldenMaster("bad-version.drv")),
"whatever",
mockXpSettings),
FormatError);
@ -56,50 +65,61 @@ TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps) {
ASSERT_THROW(
parseDerivation(
*store,
R"(Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))",
readFile(goldenMaster("bad-old-version-dyn-deps.drv")),
"dyn-dep-derivation",
mockXpSettings),
FormatError);
}
#define TEST_JSON(FIXTURE, NAME, STR, VAL, DRV_NAME, OUTPUT_NAME) \
TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _to_json) { \
using nlohmann::literals::operator "" _json; \
ASSERT_EQ( \
STR ## _json, \
(DerivationOutput { VAL }).toJSON( \
*store, \
DRV_NAME, \
OUTPUT_NAME)); \
} \
\
TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _from_json) { \
using nlohmann::literals::operator "" _json; \
ASSERT_EQ( \
DerivationOutput { VAL }, \
DerivationOutput::fromJSON( \
*store, \
DRV_NAME, \
OUTPUT_NAME, \
STR ## _json, \
mockXpSettings)); \
#define TEST_JSON(FIXTURE, NAME, VAL, DRV_NAME, OUTPUT_NAME) \
TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _from_json) { \
if (testAccept()) \
{ \
GTEST_SKIP() << cannotReadGoldenMaster; \
} \
else \
{ \
auto encoded = json::parse( \
readFile(goldenMaster("output-" #NAME ".json"))); \
DerivationOutput got = DerivationOutput::fromJSON( \
*store, \
DRV_NAME, \
OUTPUT_NAME, \
encoded, \
mockXpSettings); \
DerivationOutput expected { VAL }; \
ASSERT_EQ(got, expected); \
} \
} \
\
TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _to_json) { \
auto file = goldenMaster("output-" #NAME ".json"); \
\
json got = DerivationOutput { VAL }.toJSON( \
*store, \
DRV_NAME, \
OUTPUT_NAME); \
\
if (testAccept()) \
{ \
createDirs(dirOf(file)); \
writeFile(file, got.dump(2) + "\n"); \
GTEST_SKIP() << updatingGoldenMaster; \
} \
else \
{ \
auto expected = json::parse(readFile(file)); \
ASSERT_EQ(got, expected); \
} \
}
TEST_JSON(DerivationTest, inputAddressed,
R"({
"path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"
})",
(DerivationOutput::InputAddressed {
.path = store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"),
}),
"drv-name", "output-name")
TEST_JSON(DerivationTest, caFixedFlat,
R"({
"hashAlgo": "sha256",
"hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
"path": "/nix/store/rhcg9h16sqvlbpsa6dqm57sbr2al6nzg-drv-name-output-name"
})",
(DerivationOutput::CAFixed {
.ca = {
.method = FileIngestionMethod::Flat,
@ -109,11 +129,6 @@ TEST_JSON(DerivationTest, caFixedFlat,
"drv-name", "output-name")
TEST_JSON(DerivationTest, caFixedNAR,
R"({
"hashAlgo": "r:sha256",
"hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
"path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"
})",
(DerivationOutput::CAFixed {
.ca = {
.method = FileIngestionMethod::Recursive,
@ -123,11 +138,6 @@ TEST_JSON(DerivationTest, caFixedNAR,
"drv-name", "output-name")
TEST_JSON(DynDerivationTest, caFixedText,
R"({
"hashAlgo": "text:sha256",
"hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f",
"path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name"
})",
(DerivationOutput::CAFixed {
.ca = {
.hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="),
@ -136,9 +146,6 @@ TEST_JSON(DynDerivationTest, caFixedText,
"drv-name", "output-name")
TEST_JSON(CaDerivationTest, caFloating,
R"({
"hashAlgo": "r:sha256"
})",
(DerivationOutput::CAFloating {
.method = FileIngestionMethod::Recursive,
.hashType = htSHA256,
@ -146,15 +153,10 @@ TEST_JSON(CaDerivationTest, caFloating,
"drv-name", "output-name")
TEST_JSON(DerivationTest, deferred,
R"({ })",
DerivationOutput::Deferred { },
"drv-name", "output-name")
TEST_JSON(ImpureDerivationTest, impure,
R"({
"hashAlgo": "r:sha256",
"impure": true
})",
(DerivationOutput::Impure {
.method = FileIngestionMethod::Recursive,
.hashType = htSHA256,
@ -163,43 +165,79 @@ TEST_JSON(ImpureDerivationTest, impure,
#undef TEST_JSON
#define TEST_JSON(FIXTURE, NAME, STR, VAL) \
TEST_F(FIXTURE, Derivation_ ## NAME ## _to_json) { \
using nlohmann::literals::operator "" _json; \
ASSERT_EQ( \
STR ## _json, \
(VAL).toJSON(*store)); \
} \
\
TEST_F(FIXTURE, Derivation_ ## NAME ## _from_json) { \
using nlohmann::literals::operator "" _json; \
ASSERT_EQ( \
(VAL), \
Derivation::fromJSON( \
*store, \
STR ## _json, \
mockXpSettings)); \
#define TEST_JSON(FIXTURE, NAME, VAL) \
TEST_F(FIXTURE, Derivation_ ## NAME ## _from_json) { \
if (testAccept()) \
{ \
GTEST_SKIP() << cannotReadGoldenMaster; \
} \
else \
{ \
auto encoded = json::parse( \
readFile(goldenMaster( #NAME ".json"))); \
Derivation expected { VAL }; \
Derivation got = Derivation::fromJSON( \
*store, \
encoded, \
mockXpSettings); \
ASSERT_EQ(got, expected); \
} \
} \
\
TEST_F(FIXTURE, Derivation_ ## NAME ## _to_json) { \
auto file = goldenMaster( #NAME ".json"); \
\
json got = Derivation { VAL }.toJSON(*store); \
\
if (testAccept()) \
{ \
createDirs(dirOf(file)); \
writeFile(file, got.dump(2) + "\n"); \
GTEST_SKIP() << updatingGoldenMaster; \
} \
else \
{ \
auto expected = json::parse(readFile(file)); \
ASSERT_EQ(got, expected); \
} \
}
#define TEST_ATERM(FIXTURE, NAME, STR, VAL, DRV_NAME) \
TEST_F(FIXTURE, Derivation_ ## NAME ## _to_aterm) { \
ASSERT_EQ( \
STR, \
(VAL).unparse(*store, false)); \
} \
\
TEST_F(FIXTURE, Derivation_ ## NAME ## _from_aterm) { \
auto parsed = parseDerivation( \
*store, \
STR, \
DRV_NAME, \
mockXpSettings); \
ASSERT_EQ( \
(VAL).toJSON(*store), \
parsed.toJSON(*store)); \
ASSERT_EQ( \
(VAL), \
parsed); \
#define TEST_ATERM(FIXTURE, NAME, VAL, DRV_NAME) \
TEST_F(FIXTURE, Derivation_ ## NAME ## _from_aterm) { \
if (testAccept()) \
{ \
GTEST_SKIP() << cannotReadGoldenMaster; \
} \
else \
{ \
auto encoded = readFile(goldenMaster( #NAME ".drv")); \
Derivation expected { VAL }; \
auto got = parseDerivation( \
*store, \
std::move(encoded), \
DRV_NAME, \
mockXpSettings); \
ASSERT_EQ(got.toJSON(*store), expected.toJSON(*store)) ; \
ASSERT_EQ(got, expected); \
} \
} \
\
TEST_F(FIXTURE, Derivation_ ## NAME ## _to_aterm) { \
auto file = goldenMaster( #NAME ".drv"); \
\
auto got = (VAL).unparse(*store, false); \
\
if (testAccept()) \
{ \
createDirs(dirOf(file)); \
writeFile(file, got); \
GTEST_SKIP() << updatingGoldenMaster; \
} \
else \
{ \
auto expected = readFile(file); \
ASSERT_EQ(got, expected); \
} \
}
Derivation makeSimpleDrv(const Store & store) {
@ -236,36 +274,9 @@ Derivation makeSimpleDrv(const Store & store) {
return drv;
}
TEST_JSON(DerivationTest, simple,
R"({
"name": "simple-derivation",
"inputSrcs": [
"/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"
],
"inputDrvs": {
"/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": {
"dynamicOutputs": {},
"outputs": [
"cat",
"dog"
]
}
},
"system": "wasm-sel4",
"builder": "foo",
"args": [
"bar",
"baz"
],
"env": {
"BIG_BAD": "WOLF"
},
"outputs": {}
})",
makeSimpleDrv(*store))
TEST_JSON(DerivationTest, simple, makeSimpleDrv(*store))
TEST_ATERM(DerivationTest, simple,
R"(Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))",
makeSimpleDrv(*store),
"simple-derivation")
@ -321,45 +332,9 @@ Derivation makeDynDepDerivation(const Store & store) {
return drv;
}
TEST_JSON(DynDerivationTest, dynDerivationDeps,
R"({
"name": "dyn-dep-derivation",
"inputSrcs": [
"/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"
],
"inputDrvs": {
"/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": {
"dynamicOutputs": {
"cat": {
"dynamicOutputs": {},
"outputs": ["kitten"]
},
"goose": {
"dynamicOutputs": {},
"outputs": ["gosling"]
}
},
"outputs": [
"cat",
"dog"
]
}
},
"system": "wasm-sel4",
"builder": "foo",
"args": [
"bar",
"baz"
],
"env": {
"BIG_BAD": "WOLF"
},
"outputs": {}
})",
makeDynDepDerivation(*store))
TEST_JSON(DynDerivationTest, dynDerivationDeps, makeDynDepDerivation(*store))
TEST_ATERM(DynDerivationTest, dynDerivationDeps,
R"(DrvWithVersion("xp-dyn-drv",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))",
makeDynDepDerivation(*store),
"dyn-dep-derivation")

Some files were not shown because too many files have changed in this diff Show more